diff --git a/content/content.go b/content/content.go index 30981a1..0aa7a56 100644 --- a/content/content.go +++ b/content/content.go @@ -24,10 +24,12 @@ const ( const ( BlobStatePresent = "present" + BlobStateCorrupt = "corrupt" BlobRoleAuthoritative = "authoritative" BlobTreeStateReady = "ready" BlobEncryptionNone = "none" BlobLocationPresent = "present" + BlobLocationMissing = "missing" ) const blobUnitSize = 1024 * 1024 @@ -66,7 +68,11 @@ var ( ErrNoContentKind = errors.New("type de document sans contenu") ErrNoStoragePolicy = errors.New("politique de stockage introuvable") ErrBlobUnavailable = errors.New("blob indisponible") + ErrBlobCorrupt = errors.New("blob corrompu") ErrVersionNotFound = errors.New("version introuvable") + ErrVersionMismatch = errors.New("version associée à un autre document") + ErrVersionIsHead = errors.New("version tête de document") ErrContentPurged = errors.New("contenu purgé") ErrContentUnavailable = errors.New("contenu indisponible") + ErrInvalidRetention = errors.New("politique de rétention invalide") ) diff --git a/content/head.go b/content/head.go new file mode 100644 index 0000000..67ee7e9 --- /dev/null +++ b/content/head.go @@ -0,0 +1,106 @@ +package content + +import ( + "context" + "database/sql" + "fmt" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" +) + +func (s *Service) SetHeadVersion(ctx context.Context, actorID, documentID, versionID ident.UUID) (*Version, error) { + documentRow, err := s.getDocument(ctx, documentID) + if err != nil { + return nil, err + } + if documentRow.DeletedAt.Valid { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + if err := s.can(ctx, actorID, documentID, permission.Write); err != nil { + return nil, err + } + + version, err := s.getVersion(ctx, versionID) + if err != nil { + return nil, err + } + if version.DocumentID != documentID { + return nil, fmt.Errorf("%w: %s", ErrVersionMismatch, versionID) + } + if version.ContentPurgedAt.Valid { + return nil, fmt.Errorf("%w: %s", ErrContentPurged, versionID) + } + if documentRow.HeadVersionID != nil && *documentRow.HeadVersionID == versionID { + return versionFromDB(*version), nil + } + + rows, err := s.db.WriteQueries.SetDocumentHeadVersion(ctx, database.SetDocumentHeadVersionParams{ + HeadVersionID: &versionID, + UpdatedAt: nowMillis(), + ID: documentID, + }) + if err != nil { + return nil, err + } + if rows == 0 { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + + return versionFromDB(*version), nil +} + +func (s *Service) PurgeVersion(ctx context.Context, actorID, versionID ident.UUID) error { + version, err := s.getVersion(ctx, versionID) + if err != nil { + return err + } + + documentRow, err := s.getDocument(ctx, version.DocumentID) + if err != nil { + return err + } + if documentRow.DeletedAt.Valid { + return fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentRow.ID) + } + if err := s.can(ctx, actorID, version.DocumentID, permission.Delete); err != nil { + return err + } + if version.ContentPurgedAt.Valid { + return nil + } + if documentRow.HeadVersionID != nil && *documentRow.HeadVersionID == version.ID { + return fmt.Errorf("%w: %s", ErrVersionIsHead, version.ID) + } + + rows, err := s.db.WriteQueries.PurgeDocumentVersion(ctx, database.PurgeDocumentVersionParams{ + PurgedAt: sql.NullInt64{Int64: nowMillis(), Valid: true}, + ID: version.ID, + }) + if err != nil { + return err + } + if rows == 0 { + current, getErr := s.getVersion(ctx, version.ID) + if getErr == nil && current.ContentPurgedAt.Valid { + return nil + } + return fmt.Errorf("%w: %s", ErrVersionIsHead, version.ID) + } + + return nil +} + +func (s *Service) purgeVersion(ctx context.Context, versionID ident.UUID) (bool, error) { + rows, err := s.db.WriteQueries.PurgeDocumentVersion(ctx, database.PurgeDocumentVersionParams{ + PurgedAt: sql.NullInt64{Int64: nowMillis(), Valid: true}, + ID: versionID, + }) + if err != nil { + return false, err + } + + return rows > 0, nil +} diff --git a/content/helpers_test.go b/content/helpers_test.go index 01aac51..5818117 100644 --- a/content/helpers_test.go +++ b/content/helpers_test.go @@ -42,6 +42,7 @@ type testEnv struct { db *database.DatabaseClient store *storage.Local owner *identity.Principal + library *document.Document libraryID ident.UUID videoID ident.UUID } @@ -106,6 +107,7 @@ func newTestEnv(t *testing.T, authorizer document.Authorizer) *testEnv { db: db, store: store, owner: owner, + library: library, libraryID: library.ID, videoID: videoID, } diff --git a/content/maintenance.go b/content/maintenance.go new file mode 100644 index 0000000..b0fe083 --- /dev/null +++ b/content/maintenance.go @@ -0,0 +1,175 @@ +package content + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" + "mediarr/storage" +) + +type GarbageCollectionResult struct { + Blobs int + Locations int + Bytes int64 +} + +func (s *Service) CollectGarbage(ctx context.Context) (GarbageCollectionResult, error) { + if s.store == nil { + return GarbageCollectionResult{}, ErrStorageNotConfigured + } + + var blobs []database.Blob + var locations []database.BlobLocation + var totalBytes int64 + + err := s.withTx(ctx, func(q *database.Queries) error { + var err error + blobs, err = q.ListUnreferencedBlobs(ctx) + if err != nil { + return err + } + + for _, blob := range blobs { + totalBytes += blob.Size + + blobLocations, err := q.ListBlobLocations(ctx, blob.Root) + if err != nil { + return err + } + locations = append(locations, blobLocations...) + + if err := q.DeleteBlob(ctx, blob.Root); err != nil { + return err + } + } + + return nil + }) + if err != nil { + return GarbageCollectionResult{}, err + } + + result := GarbageCollectionResult{ + Blobs: len(blobs), + Bytes: totalBytes, + } + + var removeErrors []error + for _, location := range locations { + if err := s.store.Remove(ctx, location.Locator); err != nil { + removeErrors = append(removeErrors, err) + continue + } + result.Locations++ + } + + if len(removeErrors) > 0 { + return result, errors.Join(removeErrors...) + } + + return result, nil +} + +func (s *Service) VerifyVersion(ctx context.Context, actorID, versionID ident.UUID) error { + if s.store == nil { + return ErrStorageNotConfigured + } + + version, err := s.getVersion(ctx, versionID) + if err != nil { + return err + } + + documentRow, err := s.getDocument(ctx, version.DocumentID) + if err != nil { + return err + } + if documentRow.DeletedAt.Valid { + return fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentRow.ID) + } + if err := s.can(ctx, actorID, version.DocumentID, permission.Read); err != nil { + return err + } + if version.ContentPurgedAt.Valid { + return fmt.Errorf("%w: %s", ErrContentPurged, versionID) + } + + if _, err := s.db.Queries.GetBlob(ctx, version.ContentRoot); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + return err + } + + location, err := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: s.backendID, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + return err + } + if location.State != BlobLocationPresent { + return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + + reader, err := s.store.Open(ctx, location.Locator) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + unavailable := fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + updateErr := s.db.WriteQueries.UpdateBlobLocationState(ctx, database.UpdateBlobLocationStateParams{ + State: BlobLocationMissing, + Root: version.ContentRoot, + BackendID: s.backendID, + }) + if updateErr != nil { + return errors.Join(unavailable, updateErr) + } + return unavailable + } + return err + } + + hasher := sha256.New() + size, readErr := io.Copy(hasher, reader) + closeErr := reader.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + + now := nowMillis() + if size != version.Size || !bytes.Equal(hasher.Sum(nil), version.ContentRoot) { + corrupt := fmt.Errorf("%w: %s", ErrBlobCorrupt, versionID) + updateErr := s.db.WriteQueries.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{ + State: BlobStateCorrupt, + LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true}, + VerifyError: sql.NullString{String: "empreinte ou taille incohérente", Valid: true}, + Root: version.ContentRoot, + }) + if updateErr != nil { + return errors.Join(corrupt, updateErr) + } + return corrupt + } + + return s.db.WriteQueries.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{ + State: BlobStatePresent, + LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true}, + VerifyError: sql.NullString{}, + Root: version.ContentRoot, + }) +} diff --git a/content/p2_test.go b/content/p2_test.go new file mode 100644 index 0000000..f554067 --- /dev/null +++ b/content/p2_test.go @@ -0,0 +1,377 @@ +package content + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" + "mediarr/storage" +) + +func TestSetHeadVersionAndPurge(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + first := createTestVersion(t, env, env.videoID, "un") + second := createTestVersion(t, env, env.videoID, "deux") + + if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); err != nil { + t.Fatalf("SetHeadVersion : %v", err) + } + + documentRow, err := env.db.Queries.GetDocument(ctx, env.videoID) + if err != nil { + t.Fatalf("GetDocument : %v", err) + } + if documentRow.HeadVersionID == nil || *documentRow.HeadVersionID != first.ID { + t.Fatalf("head_version_id = %v, attendu %s", documentRow.HeadVersionID, first.ID) + } + + rev := documentRow.Rev + if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); err != nil { + t.Fatalf("SetHeadVersion(idempotent) : %v", err) + } + documentRow, err = env.db.Queries.GetDocument(ctx, env.videoID) + if err != nil { + t.Fatalf("GetDocument : %v", err) + } + if documentRow.Rev != rev { + t.Fatalf("rev = %d, attendu %d", documentRow.Rev, rev) + } + + otherVideoID := createVideo(t, env.db, env.owner, env.library, "autre.mkv") + otherVersion := createTestVersion(t, env, otherVideoID, "autre") + if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, otherVersion.ID); !errors.Is(err, ErrVersionMismatch) { + t.Fatalf("SetHeadVersion(mauvais document) : erreur = %v, attendu ErrVersionMismatch", err) + } + + if err := env.service.PurgeVersion(ctx, env.owner.ID, second.ID); err != nil { + t.Fatalf("PurgeVersion : %v", err) + } + if err := env.service.PurgeVersion(ctx, env.owner.ID, second.ID); err != nil { + t.Fatalf("PurgeVersion(idempotent) : %v", err) + } + if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, second.ID); !errors.Is(err, ErrContentPurged) { + t.Fatalf("SetHeadVersion(purgée) : erreur = %v, attendu ErrContentPurged", err) + } + if err := env.service.PurgeVersion(ctx, env.owner.ID, first.ID); !errors.Is(err, ErrVersionIsHead) { + t.Fatalf("PurgeVersion(tête) : erreur = %v, attendu ErrVersionIsHead", err) + } + + readOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Read}) + if _, err := readOnly.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("SetHeadVersion sans Write : erreur = %v, attendu ErrPermissionDenied", err) + } + if err := readOnly.PurgeVersion(ctx, env.owner.ID, second.ID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("PurgeVersion sans Delete : erreur = %v, attendu ErrPermissionDenied", err) + } +} + +func TestRetentionPurgeIDs(t *testing.T) { + now := time.Date(2026, 1, 31, 12, 0, 0, 0, time.UTC).UnixMilli() + at := func(days int) int64 { + return now - int64(days)*dayMillis + } + + headID := ident.MustParse("00000000-0000-7000-8000-000000000001") + versions := []database.DocumentVersion{ + retentionVersion(t, 1, at(1000), 0, false), + retentionVersion(t, 2, at(1000), 1, false), + retentionVersion(t, 3, at(10), 0, false), + retentionVersion(t, 4, at(40), 0, false), + retentionVersion(t, 5, at(40)-60*60*1000, 0, false), + retentionVersion(t, 6, at(41), 0, false), + retentionVersion(t, 7, at(100), 0, false), + retentionVersion(t, 8, at(110), 0, false), + retentionVersion(t, 9, at(200), 0, false), + retentionVersion(t, 10, at(300), 0, false), + retentionVersion(t, 11, at(400), 0, false), + retentionVersion(t, 12, at(10), 0, true), + } + sort.Slice(versions, func(i, j int) bool { + return versions[i].CreatedAt > versions[j].CreatedAt + }) + + purged := retentionPurgeIDs(versions, &headID, retentionPolicy{ + KeepAllDays: 30, + KeepDailyDays: 60, + KeepMonthly: 2, + }, now) + + expected := map[ident.UUID]bool{ + ident.MustParse("00000000-0000-7000-8000-000000000005"): true, + ident.MustParse("00000000-0000-7000-8000-000000000008"): true, + ident.MustParse("00000000-0000-7000-8000-000000000010"): true, + ident.MustParse("00000000-0000-7000-8000-000000000011"): true, + } + if len(purged) != len(expected) { + t.Fatalf("versions purgées = %v, attendu %v", purged, expected) + } + for _, versionID := range purged { + if !expected[versionID] { + t.Fatalf("version purgée inattendue : %s", versionID) + } + } +} + +func TestApplyRetention(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + if _, err := env.db.WriteDB.ExecContext(ctx, + "UPDATE storage_policy SET version_retention = ? WHERE id = ?", + `{"keep_all_days":30,"keep_daily_days":0,"keep_monthly":0}`, testPolicyID, + ); err != nil { + t.Fatalf("UPDATE storage_policy : %v", err) + } + + first := createTestVersion(t, env, env.videoID, "un") + second := createTestVersion(t, env, env.videoID, "deux") + head := createTestVersion(t, env, env.videoID, "trois") + + now := time.Now().UnixMilli() + setVersionCreatedAt(t, env, first.ID, now-100*dayMillis) + setVersionCreatedAt(t, env, second.ID, now-40*dayMillis) + setVersionCreatedAt(t, env, head.ID, now) + + result, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("ApplyRetention : %v", err) + } + if len(result.Purged) != 2 || result.Kept != 1 { + t.Fatalf("résultat = %+v, attendu 2 purgées et 1 conservée", result) + } + + firstRow, _ := env.db.Queries.GetDocumentVersion(ctx, first.ID) + secondRow, _ := env.db.Queries.GetDocumentVersion(ctx, second.ID) + headRow, _ := env.db.Queries.GetDocumentVersion(ctx, head.ID) + if !firstRow.ContentPurgedAt.Valid || !secondRow.ContentPurgedAt.Valid { + t.Fatalf("versions anciennes non purgées : %+v %+v", firstRow, secondRow) + } + if headRow.ContentPurgedAt.Valid { + t.Fatalf("version tête purgée : %+v", headRow) + } + + again, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("ApplyRetention(idempotent) : %v", err) + } + if len(again.Purged) != 0 || again.Kept != 3 { + t.Fatalf("second résultat = %+v, attendu 0 purgée et 3 conservées", again) + } + + manageDenied := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write}) + if _, err := manageDenied.ApplyRetention(ctx, env.owner.ID, env.videoID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("ApplyRetention sans Manage : erreur = %v, attendu ErrPermissionDenied", err) + } + + if _, err := env.db.WriteDB.ExecContext(ctx, + "UPDATE storage_policy SET version_retention = ? WHERE id = ?", + `{"keep_all_days":-1}`, testPolicyID, + ); err != nil { + t.Fatalf("UPDATE storage_policy(invalide) : %v", err) + } + if _, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID); !errors.Is(err, ErrInvalidRetention) { + t.Fatalf("ApplyRetention(politique invalide) : erreur = %v, attendu ErrInvalidRetention", err) + } +} + +func TestCollectGarbage(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + purged := createTestVersion(t, env, env.videoID, "un") + kept := createTestVersion(t, env, env.videoID, "deux") + + location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: purged.ContentRoot, + BackendID: testBackendID, + }) + if err != nil { + t.Fatalf("GetBlobLocation : %v", err) + } + + if err := env.service.PurgeVersion(ctx, env.owner.ID, purged.ID); err != nil { + t.Fatalf("PurgeVersion : %v", err) + } + + result, err := env.service.CollectGarbage(ctx) + if err != nil { + t.Fatalf("CollectGarbage : %v", err) + } + if result.Blobs != 1 || result.Locations != 1 || result.Bytes != purged.Size { + t.Fatalf("résultat = %+v", result) + } + + if _, err := env.db.Queries.GetBlob(ctx, purged.ContentRoot); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("blob purgé encore présent : %v", err) + } + if _, err := env.db.Queries.GetBlob(ctx, kept.ContentRoot); err != nil { + t.Fatalf("blob conservé : %v", err) + } + if _, err := env.store.Open(ctx, location.Locator); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("fichier purgé encore présent : %v", err) + } + + withoutStore := NewService(*env.db, nil, testBackendID, allowAll()) + if _, err := withoutStore.CollectGarbage(ctx); !errors.Is(err, ErrStorageNotConfigured) { + t.Fatalf("CollectGarbage sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err) + } +} + +func TestVerifyVersion(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + version := createTestVersion(t, env, env.videoID, "contenu vérifié") + if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); err != nil { + t.Fatalf("VerifyVersion : %v", err) + } + + blob, err := env.db.Queries.GetBlob(ctx, version.ContentRoot) + if err != nil { + t.Fatalf("GetBlob : %v", err) + } + if blob.State != BlobStatePresent || !blob.LastVerifiedAt.Valid { + t.Fatalf("blob = %+v", blob) + } + + location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: testBackendID, + }) + if err != nil { + t.Fatalf("GetBlobLocation : %v", err) + } + + path := filepath.Join("store", filepath.FromSlash(location.Locator)) + if err := os.WriteFile(path, []byte("corrompu"), 0o640); err != nil { + t.Fatalf("WriteFile : %v", err) + } + if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrBlobCorrupt) { + t.Fatalf("VerifyVersion(corrompu) : erreur = %v, attendu ErrBlobCorrupt", err) + } + + blob, err = env.db.Queries.GetBlob(ctx, version.ContentRoot) + if err != nil { + t.Fatalf("GetBlob(corrompu) : %v", err) + } + if blob.State != BlobStateCorrupt || !blob.VerifyError.Valid { + t.Fatalf("blob corrompu = %+v", blob) + } + if _, _, err := env.service.OpenVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentUnavailable) { + t.Fatalf("OpenVersion(corrompu) : erreur = %v, attendu ErrContentUnavailable", err) + } + + healed, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: strings.NewReader("contenu vérifié"), + Mime: "video/mp4", + }) + if err != nil { + t.Fatalf("CreateVersion(réparation) : %v", err) + } + if !bytes.Equal(healed.ContentRoot, version.ContentRoot) { + t.Fatalf("content_root réparé = %x, attendu %x", healed.ContentRoot, version.ContentRoot) + } + if err := env.service.VerifyVersion(ctx, env.owner.ID, healed.ID); err != nil { + t.Fatalf("VerifyVersion(réparé) : %v", err) + } + reader, _, err := env.service.OpenVersion(ctx, env.owner.ID, healed.ID) + if err != nil { + t.Fatalf("OpenVersion(réparé) : %v", err) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close(réparé) : %v", err) + } + + readOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write}) + if err := readOnly.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("VerifyVersion sans Read : erreur = %v, attendu ErrPermissionDenied", err) + } +} + +func TestVerifyVersionMissingFile(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + version := createTestVersion(t, env, env.videoID, "fichier manquant") + location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: testBackendID, + }) + if err != nil { + t.Fatalf("GetBlobLocation : %v", err) + } + + path := filepath.Join("store", filepath.FromSlash(location.Locator)) + if err := os.Remove(path); err != nil { + t.Fatalf("Remove : %v", err) + } + + if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentUnavailable) { + t.Fatalf("VerifyVersion : erreur = %v, attendu ErrContentUnavailable", err) + } + + location, err = env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: testBackendID, + }) + if err != nil { + t.Fatalf("GetBlobLocation(après) : %v", err) + } + if location.State != BlobLocationMissing { + t.Fatalf("state = %q, attendu %q", location.State, BlobLocationMissing) + } +} + +func createTestVersion(t *testing.T, env *testEnv, documentID ident.UUID, payload string) *Version { + t.Helper() + + version, err := env.service.CreateVersion(context.Background(), env.owner.ID, documentID, CreateVersionParams{ + Source: strings.NewReader(payload), + Mime: "video/mp4", + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + + return version +} + +func setVersionCreatedAt(t *testing.T, env *testEnv, versionID ident.UUID, createdAt int64) { + t.Helper() + + if _, err := env.db.WriteDB.ExecContext(context.Background(), + "UPDATE document_version SET created_at = ? WHERE id = ?", + createdAt, versionID, + ); err != nil { + t.Fatalf("UPDATE created_at : %v", err) + } +} + +func retentionVersion(t *testing.T, suffix int, createdAt int64, pinned int64, purged bool) database.DocumentVersion { + t.Helper() + + version := database.DocumentVersion{ + ID: ident.MustParse(fmt.Sprintf("00000000-0000-7000-8000-%012d", suffix)), + CreatedAt: createdAt, + Pinned: pinned, + } + if purged { + version.ContentPurgedAt = sql.NullInt64{Int64: createdAt, Valid: true} + } + + return version +} diff --git a/content/retention.go b/content/retention.go new file mode 100644 index 0000000..518f475 --- /dev/null +++ b/content/retention.go @@ -0,0 +1,153 @@ +package content + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" +) + +const dayMillis int64 = 24 * 60 * 60 * 1000 + +type RetentionResult struct { + Purged []ident.UUID + Kept int +} + +type retentionPolicy struct { + KeepAllDays int `json:"keep_all_days"` + KeepDailyDays int `json:"keep_daily_days"` + KeepMonthly int `json:"keep_monthly"` +} + +func (p retentionPolicy) validate() error { + if p.KeepAllDays < 0 || p.KeepDailyDays < 0 || p.KeepMonthly < 0 { + return ErrInvalidRetention + } + + return nil +} + +func (s *Service) ApplyRetention(ctx context.Context, actorID, documentID ident.UUID) (RetentionResult, error) { + documentRow, err := s.getDocument(ctx, documentID) + if err != nil { + return RetentionResult{}, err + } + if documentRow.DeletedAt.Valid { + return RetentionResult{}, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + if err := s.can(ctx, actorID, documentID, permission.Manage); err != nil { + return RetentionResult{}, err + } + + policyBytes, err := s.db.Queries.GetEffectiveDocumentPolicy(ctx, documentID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return RetentionResult{}, fmt.Errorf("%w: %s", ErrNoStoragePolicy, documentID) + } + return RetentionResult{}, err + } + policyID, err := ident.FromBytes(policyBytes) + if err != nil { + return RetentionResult{}, fmt.Errorf("%w: %v", ErrNoStoragePolicy, err) + } + storagePolicy, err := s.db.Queries.GetStoragePolicy(ctx, policyID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return RetentionResult{}, fmt.Errorf("%w: %s", ErrNoStoragePolicy, policyID) + } + return RetentionResult{}, err + } + + var retention retentionPolicy + if err := json.Unmarshal([]byte(storagePolicy.VersionRetention), &retention); err != nil { + return RetentionResult{}, fmt.Errorf("%w: %v", ErrInvalidRetention, err) + } + if err := retention.validate(); err != nil { + return RetentionResult{}, err + } + + versions, err := s.db.Queries.ListDocumentVersions(ctx, documentID) + if err != nil { + return RetentionResult{}, err + } + + purgeIDs := retentionPurgeIDs(versions, documentRow.HeadVersionID, retention, nowMillis()) + result := RetentionResult{ + Kept: len(versions) - len(purgeIDs), + } + + for _, versionID := range purgeIDs { + purged, err := s.purgeVersion(ctx, versionID) + if err != nil { + return result, err + } + if purged { + result.Purged = append(result.Purged, versionID) + } + } + + return result, nil +} + +func retentionPurgeIDs(versions []database.DocumentVersion, headID *ident.UUID, policy retentionPolicy, now int64) []ident.UUID { + ordered := append([]database.DocumentVersion(nil), versions...) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].CreatedAt == ordered[j].CreatedAt { + return ordered[i].Seq > ordered[j].Seq + } + return ordered[i].CreatedAt > ordered[j].CreatedAt + }) + + keep := make(map[ident.UUID]bool, len(ordered)) + dailyKept := make(map[string]bool) + monthlyKept := make(map[string]bool) + + for _, version := range ordered { + if version.ContentPurgedAt.Valid || version.Pinned != 0 || (headID != nil && *headID == version.ID) { + keep[version.ID] = true + continue + } + if version.CreatedAt > now { + keep[version.ID] = true + continue + } + + age := now - version.CreatedAt + ageDays := age / dayMillis + + switch { + case policy.KeepAllDays > 0 && ageDays <= int64(policy.KeepAllDays): + keep[version.ID] = true + case policy.KeepDailyDays > 0 && ageDays <= int64(policy.KeepDailyDays): + day := time.UnixMilli(version.CreatedAt).UTC().Format("2006-01-02") + if !dailyKept[day] { + dailyKept[day] = true + keep[version.ID] = true + } + case policy.KeepMonthly > 0: + month := time.UnixMilli(version.CreatedAt).UTC().Format("2006-01") + if !monthlyKept[month] && len(monthlyKept) < policy.KeepMonthly { + monthlyKept[month] = true + keep[version.ID] = true + } + } + } + + purge := make([]ident.UUID, 0, len(ordered)-len(keep)) + for _, version := range ordered { + if !keep[version.ID] { + purge = append(purge, version.ID) + } + } + + return purge +} diff --git a/content/version.go b/content/version.go index d53f668..3c9daef 100644 --- a/content/version.go +++ b/content/version.go @@ -107,21 +107,47 @@ func (s *Service) CreateVersion(ctx context.Context, actorID, documentID ident.U blob, err := s.db.Queries.GetBlob(ctx, staged.Root) newBlob := false newLocation := false + healBlob := false blobLocator := "" switch { case err == nil: - if blob.State != BlobStatePresent { - _ = s.store.Remove(ctx, staged.Locator) - _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) - return nil, fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) - } if blob.Size != staged.Size { _ = s.store.Remove(ctx, staged.Locator) _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) return nil, fmt.Errorf("%w: taille incohérente", ErrBlobUnavailable) } + if blob.State != BlobStatePresent { + _, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: staged.Root, + BackendID: s.backendID, + }) + switch { + case locationErr == nil: + blobLocator, err = s.store.Replace(ctx, staged) + if err != nil { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + case errors.Is(locationErr, sql.ErrNoRows): + blobLocator, err = s.store.Replace(ctx, staged) + if err != nil { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + newLocation = true + default: + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, locationErr + } + healBlob = true + break + } + location, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ Root: staged.Root, BackendID: s.backendID, @@ -193,6 +219,47 @@ func (s *Service) CreateVersion(ctx context.Context, actorID, documentID ident.U return err } } + if healBlob { + if err := q.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{ + State: BlobStatePresent, + LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true}, + VerifyError: sql.NullString{}, + Root: staged.Root, + }); err != nil { + return err + } + if !newLocation { + if err := q.UpdateBlobLocationState(ctx, database.UpdateBlobLocationStateParams{ + State: BlobLocationPresent, + Root: staged.Root, + BackendID: s.backendID, + }); err != nil { + return err + } + } + } + + if !newBlob { + existing, err := q.GetBlob(ctx, staged.Root) + if err != nil { + return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + if existing.State != BlobStatePresent || existing.Size != staged.Size { + return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + } + if !newLocation { + existingLocation, err := q.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: staged.Root, + BackendID: s.backendID, + }) + if err != nil { + return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + if existingLocation.State != BlobLocationPresent { + return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + } seq, err := q.NextDocumentVersionSeq(ctx, documentID) if err != nil { diff --git a/database/query.sql.go b/database/query.sql.go index 0597ed3..db5a242 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -653,6 +653,16 @@ func (q *Queries) DeleteAccount(ctx context.Context, principalID ident.UUID) err return err } +const deleteBlob = `-- name: DeleteBlob :exec +DELETE FROM blob +WHERE root = ?1 +` + +func (q *Queries) DeleteBlob(ctx context.Context, root []byte) error { + _, err := q.db.ExecContext(ctx, deleteBlob, root) + return err +} + const deleteExpiredSessions = `-- name: DeleteExpiredSessions :execrows DELETE FROM session WHERE expires_at <= ?1 @@ -1853,6 +1863,60 @@ func (q *Queries) ListSessionsByPrincipal(ctx context.Context, principalID ident return items, nil } +const listUnreferencedBlobs = `-- name: ListUnreferencedBlobs :many +SELECT b.root, b.size, b.state, b.role, b.unit_size, b.have, b.tree_state, b.encryption, b.last_verified_at, b.verify_error, b.last_access_at, b.created_at +FROM blob b +WHERE NOT EXISTS ( + SELECT 1 + FROM document_version v + WHERE v.content_root = b.root + AND v.content_purged_at IS NULL +) +AND NOT EXISTS ( + SELECT 1 + FROM export_link e + WHERE e.blob_root = b.root + AND e.state <> 'stale' +) +ORDER BY b.created_at +` + +func (q *Queries) ListUnreferencedBlobs(ctx context.Context) ([]Blob, error) { + rows, err := q.db.QueryContext(ctx, listUnreferencedBlobs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Blob + for rows.Next() { + var i Blob + if err := rows.Scan( + &i.Root, + &i.Size, + &i.State, + &i.Role, + &i.UnitSize, + &i.Have, + &i.TreeState, + &i.Encryption, + &i.LastVerifiedAt, + &i.VerifyError, + &i.LastAccessAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const lockAccount = `-- name: LockAccount :one UPDATE account SET state = 'locked', @@ -1953,6 +2017,32 @@ func (q *Queries) NextDocumentVersionSeq(ctx context.Context, documentID ident.U return seq, err } +const purgeDocumentVersion = `-- name: PurgeDocumentVersion :execrows +UPDATE document_version +SET content_purged_at = ?1 +WHERE document_version.id = ?2 + AND document_version.content_purged_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM document d + WHERE d.id = document_version.document_id + AND d.head_version_id = document_version.id + ) +` + +type PurgeDocumentVersionParams struct { + PurgedAt sql.NullInt64 + ID ident.UUID +} + +func (q *Queries) PurgeDocumentVersion(ctx context.Context, arg PurgeDocumentVersionParams) (int64, error) { + result, err := q.db.ExecContext(ctx, purgeDocumentVersion, arg.PurgedAt, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const recordLogin = `-- name: RecordLogin :one UPDATE account SET last_login_at = ?1, @@ -2322,6 +2412,49 @@ func (q *Queries) UpdateAccountState(ctx context.Context, arg UpdateAccountState return i, err } +const updateBlobLocationState = `-- name: UpdateBlobLocationState :exec +UPDATE blob_location +SET state = ?1 +WHERE root = ?2 + AND backend_id = ?3 +` + +type UpdateBlobLocationStateParams struct { + State string + Root []byte + BackendID ident.UUID +} + +func (q *Queries) UpdateBlobLocationState(ctx context.Context, arg UpdateBlobLocationStateParams) error { + _, err := q.db.ExecContext(ctx, updateBlobLocationState, arg.State, arg.Root, arg.BackendID) + return err +} + +const updateBlobVerification = `-- name: UpdateBlobVerification :exec +UPDATE blob +SET state = ?1, + last_verified_at = ?2, + verify_error = ?3 +WHERE root = ?4 +` + +type UpdateBlobVerificationParams struct { + State string + LastVerifiedAt sql.NullInt64 + VerifyError sql.NullString + Root []byte +} + +func (q *Queries) UpdateBlobVerification(ctx context.Context, arg UpdateBlobVerificationParams) error { + _, err := q.db.ExecContext(ctx, updateBlobVerification, + arg.State, + arg.LastVerifiedAt, + arg.VerifyError, + arg.Root, + ) + return err +} + const updateDocumentLocation = `-- name: UpdateDocumentLocation :exec UPDATE document SET path = ?1, diff --git a/database/query/query.sql b/database/query/query.sql index 7206b72..7dc2095 100644 --- a/database/query/query.sql +++ b/database/query/query.sql @@ -431,6 +431,40 @@ FROM blob_location WHERE root = sqlc.arg(root) ORDER BY created_at; +-- name: ListUnreferencedBlobs :many +SELECT b.* +FROM blob b +WHERE NOT EXISTS ( + SELECT 1 + FROM document_version v + WHERE v.content_root = b.root + AND v.content_purged_at IS NULL +) +AND NOT EXISTS ( + SELECT 1 + FROM export_link e + WHERE e.blob_root = b.root + AND e.state <> 'stale' +) +ORDER BY b.created_at; + +-- name: DeleteBlob :exec +DELETE FROM blob +WHERE root = sqlc.arg(root); + +-- name: UpdateBlobVerification :exec +UPDATE blob +SET state = sqlc.arg(state), + last_verified_at = sqlc.arg(last_verified_at), + verify_error = sqlc.narg(verify_error) +WHERE root = sqlc.arg(root); + +-- name: UpdateBlobLocationState :exec +UPDATE blob_location +SET state = sqlc.arg(state) +WHERE root = sqlc.arg(root) + AND backend_id = sqlc.arg(backend_id); + -- name: CreateBlobLocation :exec INSERT INTO blob_location ( root, @@ -680,6 +714,18 @@ SET head_version_id = sqlc.arg(head_version_id), WHERE id = sqlc.arg(id) AND deleted_at IS NULL; +-- name: PurgeDocumentVersion :execrows +UPDATE document_version +SET content_purged_at = sqlc.arg(purged_at) +WHERE document_version.id = sqlc.arg(id) + AND document_version.content_purged_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM document d + WHERE d.id = document_version.document_id + AND d.head_version_id = document_version.id + ); + -- ============================================================================ -- acl_entry -- ============================================================================ diff --git a/storage/storage.go b/storage/storage.go index c985212..4f1bc71 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -28,6 +28,7 @@ type Staged struct { type Store interface { Stage(ctx context.Context, id ident.UUID, src io.Reader) (Staged, error) Commit(ctx context.Context, staged Staged) (string, error) + Replace(ctx context.Context, staged Staged) (string, error) Open(ctx context.Context, locator string) (io.ReadCloser, error) Remove(ctx context.Context, locator string) error } @@ -136,6 +137,36 @@ func (l *Local) Commit(ctx context.Context, staged Staged) (string, error) { return locator, nil } +func (l *Local) Replace(ctx context.Context, staged Staged) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if len(staged.Root) != sha256.Size { + return "", fmt.Errorf("%w: empreinte de %d octets", ErrInvalidLocator, len(staged.Root)) + } + + source, err := l.resolve(staged.Locator) + if err != nil { + return "", err + } + + root := hex.EncodeToString(staged.Root) + locator := filepath.ToSlash(filepath.Join("blobs", root[:2], root[2:4], root)) + target, err := l.resolve(locator) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return "", err + } + + if err := os.Rename(source, target); err != nil { + return "", err + } + + return locator, nil +} + func (l *Local) Open(ctx context.Context, locator string) (io.ReadCloser, error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/storage/storage_test.go b/storage/storage_test.go index f774ac5..7412124 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -94,6 +94,58 @@ func TestLocalStageCommitDedupOpenRemove(t *testing.T) { } } +func TestLocalReplace(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + + store, err := NewLocal(root) + if err != nil { + t.Fatalf("NewLocal : %v", err) + } + + payload := []byte("contenu à réparer") + staged, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Stage : %v", err) + } + locator, err := store.Commit(ctx, staged) + if err != nil { + t.Fatalf("Commit : %v", err) + } + + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(locator)), []byte("corrompu"), 0o640); err != nil { + t.Fatalf("WriteFile : %v", err) + } + + replacement, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Stage(remplacement) : %v", err) + } + replacedLocator, err := store.Replace(ctx, replacement) + if err != nil { + t.Fatalf("Replace : %v", err) + } + if replacedLocator != locator { + t.Fatalf("locator = %q, attendu %q", replacedLocator, locator) + } + + reader, err := store.Open(ctx, locator) + if err != nil { + t.Fatalf("Open : %v", err) + } + data, err := io.ReadAll(reader) + if err != nil { + _ = reader.Close() + t.Fatalf("ReadAll : %v", err) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close : %v", err) + } + if !bytes.Equal(data, payload) { + t.Fatalf("contenu = %q, attendu %q", data, payload) + } +} + func TestLocalRejectsInvalidLocators(t *testing.T) { ctx := context.Background() store, err := NewLocal(t.TempDir())