retention and replace

This commit is contained in:
venture
2026-11-16 10:04:52 +01:00
parent 610192e2d5
commit cc0ddd0dfb
11 changed files with 1153 additions and 5 deletions
+6
View File
@@ -24,10 +24,12 @@ const (
const ( const (
BlobStatePresent = "present" BlobStatePresent = "present"
BlobStateCorrupt = "corrupt"
BlobRoleAuthoritative = "authoritative" BlobRoleAuthoritative = "authoritative"
BlobTreeStateReady = "ready" BlobTreeStateReady = "ready"
BlobEncryptionNone = "none" BlobEncryptionNone = "none"
BlobLocationPresent = "present" BlobLocationPresent = "present"
BlobLocationMissing = "missing"
) )
const blobUnitSize = 1024 * 1024 const blobUnitSize = 1024 * 1024
@@ -66,7 +68,11 @@ var (
ErrNoContentKind = errors.New("type de document sans contenu") ErrNoContentKind = errors.New("type de document sans contenu")
ErrNoStoragePolicy = errors.New("politique de stockage introuvable") ErrNoStoragePolicy = errors.New("politique de stockage introuvable")
ErrBlobUnavailable = errors.New("blob indisponible") ErrBlobUnavailable = errors.New("blob indisponible")
ErrBlobCorrupt = errors.New("blob corrompu")
ErrVersionNotFound = errors.New("version introuvable") ErrVersionNotFound = errors.New("version introuvable")
ErrVersionMismatch = errors.New("version associée à un autre document")
ErrVersionIsHead = errors.New("version tête de document")
ErrContentPurged = errors.New("contenu purgé") ErrContentPurged = errors.New("contenu purgé")
ErrContentUnavailable = errors.New("contenu indisponible") ErrContentUnavailable = errors.New("contenu indisponible")
ErrInvalidRetention = errors.New("politique de rétention invalide")
) )
+106
View File
@@ -0,0 +1,106 @@
package content
import (
"context"
"database/sql"
"fmt"
"mediarr/database"
"mediarr/document"
"mediarr/ident"
"mediarr/permission"
)
func (s *Service) SetHeadVersion(ctx context.Context, actorID, documentID, versionID ident.UUID) (*Version, error) {
documentRow, err := s.getDocument(ctx, documentID)
if err != nil {
return nil, err
}
if documentRow.DeletedAt.Valid {
return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID)
}
if err := s.can(ctx, actorID, documentID, permission.Write); err != nil {
return nil, err
}
version, err := s.getVersion(ctx, versionID)
if err != nil {
return nil, err
}
if version.DocumentID != documentID {
return nil, fmt.Errorf("%w: %s", ErrVersionMismatch, versionID)
}
if version.ContentPurgedAt.Valid {
return nil, fmt.Errorf("%w: %s", ErrContentPurged, versionID)
}
if documentRow.HeadVersionID != nil && *documentRow.HeadVersionID == versionID {
return versionFromDB(*version), nil
}
rows, err := s.db.WriteQueries.SetDocumentHeadVersion(ctx, database.SetDocumentHeadVersionParams{
HeadVersionID: &versionID,
UpdatedAt: nowMillis(),
ID: documentID,
})
if err != nil {
return nil, err
}
if rows == 0 {
return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID)
}
return versionFromDB(*version), nil
}
func (s *Service) PurgeVersion(ctx context.Context, actorID, versionID ident.UUID) error {
version, err := s.getVersion(ctx, versionID)
if err != nil {
return err
}
documentRow, err := s.getDocument(ctx, version.DocumentID)
if err != nil {
return err
}
if documentRow.DeletedAt.Valid {
return fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentRow.ID)
}
if err := s.can(ctx, actorID, version.DocumentID, permission.Delete); err != nil {
return err
}
if version.ContentPurgedAt.Valid {
return nil
}
if documentRow.HeadVersionID != nil && *documentRow.HeadVersionID == version.ID {
return fmt.Errorf("%w: %s", ErrVersionIsHead, version.ID)
}
rows, err := s.db.WriteQueries.PurgeDocumentVersion(ctx, database.PurgeDocumentVersionParams{
PurgedAt: sql.NullInt64{Int64: nowMillis(), Valid: true},
ID: version.ID,
})
if err != nil {
return err
}
if rows == 0 {
current, getErr := s.getVersion(ctx, version.ID)
if getErr == nil && current.ContentPurgedAt.Valid {
return nil
}
return fmt.Errorf("%w: %s", ErrVersionIsHead, version.ID)
}
return nil
}
func (s *Service) purgeVersion(ctx context.Context, versionID ident.UUID) (bool, error) {
rows, err := s.db.WriteQueries.PurgeDocumentVersion(ctx, database.PurgeDocumentVersionParams{
PurgedAt: sql.NullInt64{Int64: nowMillis(), Valid: true},
ID: versionID,
})
if err != nil {
return false, err
}
return rows > 0, nil
}
+2
View File
@@ -42,6 +42,7 @@ type testEnv struct {
db *database.DatabaseClient db *database.DatabaseClient
store *storage.Local store *storage.Local
owner *identity.Principal owner *identity.Principal
library *document.Document
libraryID ident.UUID libraryID ident.UUID
videoID ident.UUID videoID ident.UUID
} }
@@ -106,6 +107,7 @@ func newTestEnv(t *testing.T, authorizer document.Authorizer) *testEnv {
db: db, db: db,
store: store, store: store,
owner: owner, owner: owner,
library: library,
libraryID: library.ID, libraryID: library.ID,
videoID: videoID, videoID: videoID,
} }
+175
View File
@@ -0,0 +1,175 @@
package content
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"io"
"mediarr/database"
"mediarr/document"
"mediarr/ident"
"mediarr/permission"
"mediarr/storage"
)
type GarbageCollectionResult struct {
Blobs int
Locations int
Bytes int64
}
func (s *Service) CollectGarbage(ctx context.Context) (GarbageCollectionResult, error) {
if s.store == nil {
return GarbageCollectionResult{}, ErrStorageNotConfigured
}
var blobs []database.Blob
var locations []database.BlobLocation
var totalBytes int64
err := s.withTx(ctx, func(q *database.Queries) error {
var err error
blobs, err = q.ListUnreferencedBlobs(ctx)
if err != nil {
return err
}
for _, blob := range blobs {
totalBytes += blob.Size
blobLocations, err := q.ListBlobLocations(ctx, blob.Root)
if err != nil {
return err
}
locations = append(locations, blobLocations...)
if err := q.DeleteBlob(ctx, blob.Root); err != nil {
return err
}
}
return nil
})
if err != nil {
return GarbageCollectionResult{}, err
}
result := GarbageCollectionResult{
Blobs: len(blobs),
Bytes: totalBytes,
}
var removeErrors []error
for _, location := range locations {
if err := s.store.Remove(ctx, location.Locator); err != nil {
removeErrors = append(removeErrors, err)
continue
}
result.Locations++
}
if len(removeErrors) > 0 {
return result, errors.Join(removeErrors...)
}
return result, nil
}
func (s *Service) VerifyVersion(ctx context.Context, actorID, versionID ident.UUID) error {
if s.store == nil {
return ErrStorageNotConfigured
}
version, err := s.getVersion(ctx, versionID)
if err != nil {
return err
}
documentRow, err := s.getDocument(ctx, version.DocumentID)
if err != nil {
return err
}
if documentRow.DeletedAt.Valid {
return fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentRow.ID)
}
if err := s.can(ctx, actorID, version.DocumentID, permission.Read); err != nil {
return err
}
if version.ContentPurgedAt.Valid {
return fmt.Errorf("%w: %s", ErrContentPurged, versionID)
}
if _, err := s.db.Queries.GetBlob(ctx, version.ContentRoot); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID)
}
return err
}
location, err := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: version.ContentRoot,
BackendID: s.backendID,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID)
}
return err
}
if location.State != BlobLocationPresent {
return fmt.Errorf("%w: %s", ErrContentUnavailable, versionID)
}
reader, err := s.store.Open(ctx, location.Locator)
if err != nil {
if errors.Is(err, storage.ErrNotFound) {
unavailable := fmt.Errorf("%w: %s", ErrContentUnavailable, versionID)
updateErr := s.db.WriteQueries.UpdateBlobLocationState(ctx, database.UpdateBlobLocationStateParams{
State: BlobLocationMissing,
Root: version.ContentRoot,
BackendID: s.backendID,
})
if updateErr != nil {
return errors.Join(unavailable, updateErr)
}
return unavailable
}
return err
}
hasher := sha256.New()
size, readErr := io.Copy(hasher, reader)
closeErr := reader.Close()
if readErr != nil {
return readErr
}
if closeErr != nil {
return closeErr
}
now := nowMillis()
if size != version.Size || !bytes.Equal(hasher.Sum(nil), version.ContentRoot) {
corrupt := fmt.Errorf("%w: %s", ErrBlobCorrupt, versionID)
updateErr := s.db.WriteQueries.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{
State: BlobStateCorrupt,
LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true},
VerifyError: sql.NullString{String: "empreinte ou taille incohérente", Valid: true},
Root: version.ContentRoot,
})
if updateErr != nil {
return errors.Join(corrupt, updateErr)
}
return corrupt
}
return s.db.WriteQueries.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{
State: BlobStatePresent,
LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true},
VerifyError: sql.NullString{},
Root: version.ContentRoot,
})
}
+377
View File
@@ -0,0 +1,377 @@
package content
import (
"bytes"
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"mediarr/database"
"mediarr/document"
"mediarr/ident"
"mediarr/permission"
"mediarr/storage"
)
func TestSetHeadVersionAndPurge(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
first := createTestVersion(t, env, env.videoID, "un")
second := createTestVersion(t, env, env.videoID, "deux")
if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); err != nil {
t.Fatalf("SetHeadVersion : %v", err)
}
documentRow, err := env.db.Queries.GetDocument(ctx, env.videoID)
if err != nil {
t.Fatalf("GetDocument : %v", err)
}
if documentRow.HeadVersionID == nil || *documentRow.HeadVersionID != first.ID {
t.Fatalf("head_version_id = %v, attendu %s", documentRow.HeadVersionID, first.ID)
}
rev := documentRow.Rev
if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); err != nil {
t.Fatalf("SetHeadVersion(idempotent) : %v", err)
}
documentRow, err = env.db.Queries.GetDocument(ctx, env.videoID)
if err != nil {
t.Fatalf("GetDocument : %v", err)
}
if documentRow.Rev != rev {
t.Fatalf("rev = %d, attendu %d", documentRow.Rev, rev)
}
otherVideoID := createVideo(t, env.db, env.owner, env.library, "autre.mkv")
otherVersion := createTestVersion(t, env, otherVideoID, "autre")
if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, otherVersion.ID); !errors.Is(err, ErrVersionMismatch) {
t.Fatalf("SetHeadVersion(mauvais document) : erreur = %v, attendu ErrVersionMismatch", err)
}
if err := env.service.PurgeVersion(ctx, env.owner.ID, second.ID); err != nil {
t.Fatalf("PurgeVersion : %v", err)
}
if err := env.service.PurgeVersion(ctx, env.owner.ID, second.ID); err != nil {
t.Fatalf("PurgeVersion(idempotent) : %v", err)
}
if _, err := env.service.SetHeadVersion(ctx, env.owner.ID, env.videoID, second.ID); !errors.Is(err, ErrContentPurged) {
t.Fatalf("SetHeadVersion(purgée) : erreur = %v, attendu ErrContentPurged", err)
}
if err := env.service.PurgeVersion(ctx, env.owner.ID, first.ID); !errors.Is(err, ErrVersionIsHead) {
t.Fatalf("PurgeVersion(tête) : erreur = %v, attendu ErrVersionIsHead", err)
}
readOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Read})
if _, err := readOnly.SetHeadVersion(ctx, env.owner.ID, env.videoID, first.ID); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("SetHeadVersion sans Write : erreur = %v, attendu ErrPermissionDenied", err)
}
if err := readOnly.PurgeVersion(ctx, env.owner.ID, second.ID); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("PurgeVersion sans Delete : erreur = %v, attendu ErrPermissionDenied", err)
}
}
func TestRetentionPurgeIDs(t *testing.T) {
now := time.Date(2026, 1, 31, 12, 0, 0, 0, time.UTC).UnixMilli()
at := func(days int) int64 {
return now - int64(days)*dayMillis
}
headID := ident.MustParse("00000000-0000-7000-8000-000000000001")
versions := []database.DocumentVersion{
retentionVersion(t, 1, at(1000), 0, false),
retentionVersion(t, 2, at(1000), 1, false),
retentionVersion(t, 3, at(10), 0, false),
retentionVersion(t, 4, at(40), 0, false),
retentionVersion(t, 5, at(40)-60*60*1000, 0, false),
retentionVersion(t, 6, at(41), 0, false),
retentionVersion(t, 7, at(100), 0, false),
retentionVersion(t, 8, at(110), 0, false),
retentionVersion(t, 9, at(200), 0, false),
retentionVersion(t, 10, at(300), 0, false),
retentionVersion(t, 11, at(400), 0, false),
retentionVersion(t, 12, at(10), 0, true),
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].CreatedAt > versions[j].CreatedAt
})
purged := retentionPurgeIDs(versions, &headID, retentionPolicy{
KeepAllDays: 30,
KeepDailyDays: 60,
KeepMonthly: 2,
}, now)
expected := map[ident.UUID]bool{
ident.MustParse("00000000-0000-7000-8000-000000000005"): true,
ident.MustParse("00000000-0000-7000-8000-000000000008"): true,
ident.MustParse("00000000-0000-7000-8000-000000000010"): true,
ident.MustParse("00000000-0000-7000-8000-000000000011"): true,
}
if len(purged) != len(expected) {
t.Fatalf("versions purgées = %v, attendu %v", purged, expected)
}
for _, versionID := range purged {
if !expected[versionID] {
t.Fatalf("version purgée inattendue : %s", versionID)
}
}
}
func TestApplyRetention(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
if _, err := env.db.WriteDB.ExecContext(ctx,
"UPDATE storage_policy SET version_retention = ? WHERE id = ?",
`{"keep_all_days":30,"keep_daily_days":0,"keep_monthly":0}`, testPolicyID,
); err != nil {
t.Fatalf("UPDATE storage_policy : %v", err)
}
first := createTestVersion(t, env, env.videoID, "un")
second := createTestVersion(t, env, env.videoID, "deux")
head := createTestVersion(t, env, env.videoID, "trois")
now := time.Now().UnixMilli()
setVersionCreatedAt(t, env, first.ID, now-100*dayMillis)
setVersionCreatedAt(t, env, second.ID, now-40*dayMillis)
setVersionCreatedAt(t, env, head.ID, now)
result, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID)
if err != nil {
t.Fatalf("ApplyRetention : %v", err)
}
if len(result.Purged) != 2 || result.Kept != 1 {
t.Fatalf("résultat = %+v, attendu 2 purgées et 1 conservée", result)
}
firstRow, _ := env.db.Queries.GetDocumentVersion(ctx, first.ID)
secondRow, _ := env.db.Queries.GetDocumentVersion(ctx, second.ID)
headRow, _ := env.db.Queries.GetDocumentVersion(ctx, head.ID)
if !firstRow.ContentPurgedAt.Valid || !secondRow.ContentPurgedAt.Valid {
t.Fatalf("versions anciennes non purgées : %+v %+v", firstRow, secondRow)
}
if headRow.ContentPurgedAt.Valid {
t.Fatalf("version tête purgée : %+v", headRow)
}
again, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID)
if err != nil {
t.Fatalf("ApplyRetention(idempotent) : %v", err)
}
if len(again.Purged) != 0 || again.Kept != 3 {
t.Fatalf("second résultat = %+v, attendu 0 purgée et 3 conservées", again)
}
manageDenied := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write})
if _, err := manageDenied.ApplyRetention(ctx, env.owner.ID, env.videoID); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("ApplyRetention sans Manage : erreur = %v, attendu ErrPermissionDenied", err)
}
if _, err := env.db.WriteDB.ExecContext(ctx,
"UPDATE storage_policy SET version_retention = ? WHERE id = ?",
`{"keep_all_days":-1}`, testPolicyID,
); err != nil {
t.Fatalf("UPDATE storage_policy(invalide) : %v", err)
}
if _, err := env.service.ApplyRetention(ctx, env.owner.ID, env.videoID); !errors.Is(err, ErrInvalidRetention) {
t.Fatalf("ApplyRetention(politique invalide) : erreur = %v, attendu ErrInvalidRetention", err)
}
}
func TestCollectGarbage(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
purged := createTestVersion(t, env, env.videoID, "un")
kept := createTestVersion(t, env, env.videoID, "deux")
location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: purged.ContentRoot,
BackendID: testBackendID,
})
if err != nil {
t.Fatalf("GetBlobLocation : %v", err)
}
if err := env.service.PurgeVersion(ctx, env.owner.ID, purged.ID); err != nil {
t.Fatalf("PurgeVersion : %v", err)
}
result, err := env.service.CollectGarbage(ctx)
if err != nil {
t.Fatalf("CollectGarbage : %v", err)
}
if result.Blobs != 1 || result.Locations != 1 || result.Bytes != purged.Size {
t.Fatalf("résultat = %+v", result)
}
if _, err := env.db.Queries.GetBlob(ctx, purged.ContentRoot); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("blob purgé encore présent : %v", err)
}
if _, err := env.db.Queries.GetBlob(ctx, kept.ContentRoot); err != nil {
t.Fatalf("blob conservé : %v", err)
}
if _, err := env.store.Open(ctx, location.Locator); !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("fichier purgé encore présent : %v", err)
}
withoutStore := NewService(*env.db, nil, testBackendID, allowAll())
if _, err := withoutStore.CollectGarbage(ctx); !errors.Is(err, ErrStorageNotConfigured) {
t.Fatalf("CollectGarbage sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err)
}
}
func TestVerifyVersion(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
version := createTestVersion(t, env, env.videoID, "contenu vérifié")
if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); err != nil {
t.Fatalf("VerifyVersion : %v", err)
}
blob, err := env.db.Queries.GetBlob(ctx, version.ContentRoot)
if err != nil {
t.Fatalf("GetBlob : %v", err)
}
if blob.State != BlobStatePresent || !blob.LastVerifiedAt.Valid {
t.Fatalf("blob = %+v", blob)
}
location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: version.ContentRoot,
BackendID: testBackendID,
})
if err != nil {
t.Fatalf("GetBlobLocation : %v", err)
}
path := filepath.Join("store", filepath.FromSlash(location.Locator))
if err := os.WriteFile(path, []byte("corrompu"), 0o640); err != nil {
t.Fatalf("WriteFile : %v", err)
}
if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrBlobCorrupt) {
t.Fatalf("VerifyVersion(corrompu) : erreur = %v, attendu ErrBlobCorrupt", err)
}
blob, err = env.db.Queries.GetBlob(ctx, version.ContentRoot)
if err != nil {
t.Fatalf("GetBlob(corrompu) : %v", err)
}
if blob.State != BlobStateCorrupt || !blob.VerifyError.Valid {
t.Fatalf("blob corrompu = %+v", blob)
}
if _, _, err := env.service.OpenVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentUnavailable) {
t.Fatalf("OpenVersion(corrompu) : erreur = %v, attendu ErrContentUnavailable", err)
}
healed, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{
Source: strings.NewReader("contenu vérifié"),
Mime: "video/mp4",
})
if err != nil {
t.Fatalf("CreateVersion(réparation) : %v", err)
}
if !bytes.Equal(healed.ContentRoot, version.ContentRoot) {
t.Fatalf("content_root réparé = %x, attendu %x", healed.ContentRoot, version.ContentRoot)
}
if err := env.service.VerifyVersion(ctx, env.owner.ID, healed.ID); err != nil {
t.Fatalf("VerifyVersion(réparé) : %v", err)
}
reader, _, err := env.service.OpenVersion(ctx, env.owner.ID, healed.ID)
if err != nil {
t.Fatalf("OpenVersion(réparé) : %v", err)
}
if err := reader.Close(); err != nil {
t.Fatalf("Close(réparé) : %v", err)
}
readOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write})
if err := readOnly.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("VerifyVersion sans Read : erreur = %v, attendu ErrPermissionDenied", err)
}
}
func TestVerifyVersionMissingFile(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
version := createTestVersion(t, env, env.videoID, "fichier manquant")
location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: version.ContentRoot,
BackendID: testBackendID,
})
if err != nil {
t.Fatalf("GetBlobLocation : %v", err)
}
path := filepath.Join("store", filepath.FromSlash(location.Locator))
if err := os.Remove(path); err != nil {
t.Fatalf("Remove : %v", err)
}
if err := env.service.VerifyVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentUnavailable) {
t.Fatalf("VerifyVersion : erreur = %v, attendu ErrContentUnavailable", err)
}
location, err = env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: version.ContentRoot,
BackendID: testBackendID,
})
if err != nil {
t.Fatalf("GetBlobLocation(après) : %v", err)
}
if location.State != BlobLocationMissing {
t.Fatalf("state = %q, attendu %q", location.State, BlobLocationMissing)
}
}
func createTestVersion(t *testing.T, env *testEnv, documentID ident.UUID, payload string) *Version {
t.Helper()
version, err := env.service.CreateVersion(context.Background(), env.owner.ID, documentID, CreateVersionParams{
Source: strings.NewReader(payload),
Mime: "video/mp4",
})
if err != nil {
t.Fatalf("CreateVersion : %v", err)
}
return version
}
func setVersionCreatedAt(t *testing.T, env *testEnv, versionID ident.UUID, createdAt int64) {
t.Helper()
if _, err := env.db.WriteDB.ExecContext(context.Background(),
"UPDATE document_version SET created_at = ? WHERE id = ?",
createdAt, versionID,
); err != nil {
t.Fatalf("UPDATE created_at : %v", err)
}
}
func retentionVersion(t *testing.T, suffix int, createdAt int64, pinned int64, purged bool) database.DocumentVersion {
t.Helper()
version := database.DocumentVersion{
ID: ident.MustParse(fmt.Sprintf("00000000-0000-7000-8000-%012d", suffix)),
CreatedAt: createdAt,
Pinned: pinned,
}
if purged {
version.ContentPurgedAt = sql.NullInt64{Int64: createdAt, Valid: true}
}
return version
}
+153
View File
@@ -0,0 +1,153 @@
package content
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"sort"
"time"
"mediarr/database"
"mediarr/document"
"mediarr/ident"
"mediarr/permission"
)
const dayMillis int64 = 24 * 60 * 60 * 1000
type RetentionResult struct {
Purged []ident.UUID
Kept int
}
type retentionPolicy struct {
KeepAllDays int `json:"keep_all_days"`
KeepDailyDays int `json:"keep_daily_days"`
KeepMonthly int `json:"keep_monthly"`
}
func (p retentionPolicy) validate() error {
if p.KeepAllDays < 0 || p.KeepDailyDays < 0 || p.KeepMonthly < 0 {
return ErrInvalidRetention
}
return nil
}
func (s *Service) ApplyRetention(ctx context.Context, actorID, documentID ident.UUID) (RetentionResult, error) {
documentRow, err := s.getDocument(ctx, documentID)
if err != nil {
return RetentionResult{}, err
}
if documentRow.DeletedAt.Valid {
return RetentionResult{}, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID)
}
if err := s.can(ctx, actorID, documentID, permission.Manage); err != nil {
return RetentionResult{}, err
}
policyBytes, err := s.db.Queries.GetEffectiveDocumentPolicy(ctx, documentID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return RetentionResult{}, fmt.Errorf("%w: %s", ErrNoStoragePolicy, documentID)
}
return RetentionResult{}, err
}
policyID, err := ident.FromBytes(policyBytes)
if err != nil {
return RetentionResult{}, fmt.Errorf("%w: %v", ErrNoStoragePolicy, err)
}
storagePolicy, err := s.db.Queries.GetStoragePolicy(ctx, policyID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return RetentionResult{}, fmt.Errorf("%w: %s", ErrNoStoragePolicy, policyID)
}
return RetentionResult{}, err
}
var retention retentionPolicy
if err := json.Unmarshal([]byte(storagePolicy.VersionRetention), &retention); err != nil {
return RetentionResult{}, fmt.Errorf("%w: %v", ErrInvalidRetention, err)
}
if err := retention.validate(); err != nil {
return RetentionResult{}, err
}
versions, err := s.db.Queries.ListDocumentVersions(ctx, documentID)
if err != nil {
return RetentionResult{}, err
}
purgeIDs := retentionPurgeIDs(versions, documentRow.HeadVersionID, retention, nowMillis())
result := RetentionResult{
Kept: len(versions) - len(purgeIDs),
}
for _, versionID := range purgeIDs {
purged, err := s.purgeVersion(ctx, versionID)
if err != nil {
return result, err
}
if purged {
result.Purged = append(result.Purged, versionID)
}
}
return result, nil
}
func retentionPurgeIDs(versions []database.DocumentVersion, headID *ident.UUID, policy retentionPolicy, now int64) []ident.UUID {
ordered := append([]database.DocumentVersion(nil), versions...)
sort.Slice(ordered, func(i, j int) bool {
if ordered[i].CreatedAt == ordered[j].CreatedAt {
return ordered[i].Seq > ordered[j].Seq
}
return ordered[i].CreatedAt > ordered[j].CreatedAt
})
keep := make(map[ident.UUID]bool, len(ordered))
dailyKept := make(map[string]bool)
monthlyKept := make(map[string]bool)
for _, version := range ordered {
if version.ContentPurgedAt.Valid || version.Pinned != 0 || (headID != nil && *headID == version.ID) {
keep[version.ID] = true
continue
}
if version.CreatedAt > now {
keep[version.ID] = true
continue
}
age := now - version.CreatedAt
ageDays := age / dayMillis
switch {
case policy.KeepAllDays > 0 && ageDays <= int64(policy.KeepAllDays):
keep[version.ID] = true
case policy.KeepDailyDays > 0 && ageDays <= int64(policy.KeepDailyDays):
day := time.UnixMilli(version.CreatedAt).UTC().Format("2006-01-02")
if !dailyKept[day] {
dailyKept[day] = true
keep[version.ID] = true
}
case policy.KeepMonthly > 0:
month := time.UnixMilli(version.CreatedAt).UTC().Format("2006-01")
if !monthlyKept[month] && len(monthlyKept) < policy.KeepMonthly {
monthlyKept[month] = true
keep[version.ID] = true
}
}
}
purge := make([]ident.UUID, 0, len(ordered)-len(keep))
for _, version := range ordered {
if !keep[version.ID] {
purge = append(purge, version.ID)
}
}
return purge
}
+72 -5
View File
@@ -107,21 +107,47 @@ func (s *Service) CreateVersion(ctx context.Context, actorID, documentID ident.U
blob, err := s.db.Queries.GetBlob(ctx, staged.Root) blob, err := s.db.Queries.GetBlob(ctx, staged.Root)
newBlob := false newBlob := false
newLocation := false newLocation := false
healBlob := false
blobLocator := "" blobLocator := ""
switch { switch {
case err == nil: case err == nil:
if blob.State != BlobStatePresent {
_ = s.store.Remove(ctx, staged.Locator)
_ = s.setStagingState(ctx, stagingID, StagingStateDiscarded)
return nil, fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root)
}
if blob.Size != staged.Size { if blob.Size != staged.Size {
_ = s.store.Remove(ctx, staged.Locator) _ = s.store.Remove(ctx, staged.Locator)
_ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded)
return nil, fmt.Errorf("%w: taille incohérente", ErrBlobUnavailable) return nil, fmt.Errorf("%w: taille incohérente", ErrBlobUnavailable)
} }
if blob.State != BlobStatePresent {
_, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: staged.Root,
BackendID: s.backendID,
})
switch {
case locationErr == nil:
blobLocator, err = s.store.Replace(ctx, staged)
if err != nil {
_ = s.store.Remove(ctx, staged.Locator)
_ = s.setStagingState(ctx, stagingID, StagingStateDiscarded)
return nil, err
}
case errors.Is(locationErr, sql.ErrNoRows):
blobLocator, err = s.store.Replace(ctx, staged)
if err != nil {
_ = s.store.Remove(ctx, staged.Locator)
_ = s.setStagingState(ctx, stagingID, StagingStateDiscarded)
return nil, err
}
newLocation = true
default:
_ = s.store.Remove(ctx, staged.Locator)
_ = s.setStagingState(ctx, stagingID, StagingStateDiscarded)
return nil, locationErr
}
healBlob = true
break
}
location, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ location, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: staged.Root, Root: staged.Root,
BackendID: s.backendID, BackendID: s.backendID,
@@ -193,6 +219,47 @@ func (s *Service) CreateVersion(ctx context.Context, actorID, documentID ident.U
return err return err
} }
} }
if healBlob {
if err := q.UpdateBlobVerification(ctx, database.UpdateBlobVerificationParams{
State: BlobStatePresent,
LastVerifiedAt: sql.NullInt64{Int64: now, Valid: true},
VerifyError: sql.NullString{},
Root: staged.Root,
}); err != nil {
return err
}
if !newLocation {
if err := q.UpdateBlobLocationState(ctx, database.UpdateBlobLocationStateParams{
State: BlobLocationPresent,
Root: staged.Root,
BackendID: s.backendID,
}); err != nil {
return err
}
}
}
if !newBlob {
existing, err := q.GetBlob(ctx, staged.Root)
if err != nil {
return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root)
}
if existing.State != BlobStatePresent || existing.Size != staged.Size {
return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root)
}
}
if !newLocation {
existingLocation, err := q.GetBlobLocation(ctx, database.GetBlobLocationParams{
Root: staged.Root,
BackendID: s.backendID,
})
if err != nil {
return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root)
}
if existingLocation.State != BlobLocationPresent {
return fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root)
}
}
seq, err := q.NextDocumentVersionSeq(ctx, documentID) seq, err := q.NextDocumentVersionSeq(ctx, documentID)
if err != nil { if err != nil {
+133
View File
@@ -653,6 +653,16 @@ func (q *Queries) DeleteAccount(ctx context.Context, principalID ident.UUID) err
return err return err
} }
const deleteBlob = `-- name: DeleteBlob :exec
DELETE FROM blob
WHERE root = ?1
`
func (q *Queries) DeleteBlob(ctx context.Context, root []byte) error {
_, err := q.db.ExecContext(ctx, deleteBlob, root)
return err
}
const deleteExpiredSessions = `-- name: DeleteExpiredSessions :execrows const deleteExpiredSessions = `-- name: DeleteExpiredSessions :execrows
DELETE FROM session DELETE FROM session
WHERE expires_at <= ?1 WHERE expires_at <= ?1
@@ -1853,6 +1863,60 @@ func (q *Queries) ListSessionsByPrincipal(ctx context.Context, principalID ident
return items, nil return items, nil
} }
const listUnreferencedBlobs = `-- name: ListUnreferencedBlobs :many
SELECT b.root, b.size, b.state, b.role, b.unit_size, b.have, b.tree_state, b.encryption, b.last_verified_at, b.verify_error, b.last_access_at, b.created_at
FROM blob b
WHERE NOT EXISTS (
SELECT 1
FROM document_version v
WHERE v.content_root = b.root
AND v.content_purged_at IS NULL
)
AND NOT EXISTS (
SELECT 1
FROM export_link e
WHERE e.blob_root = b.root
AND e.state <> 'stale'
)
ORDER BY b.created_at
`
func (q *Queries) ListUnreferencedBlobs(ctx context.Context) ([]Blob, error) {
rows, err := q.db.QueryContext(ctx, listUnreferencedBlobs)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Blob
for rows.Next() {
var i Blob
if err := rows.Scan(
&i.Root,
&i.Size,
&i.State,
&i.Role,
&i.UnitSize,
&i.Have,
&i.TreeState,
&i.Encryption,
&i.LastVerifiedAt,
&i.VerifyError,
&i.LastAccessAt,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockAccount = `-- name: LockAccount :one const lockAccount = `-- name: LockAccount :one
UPDATE account UPDATE account
SET state = 'locked', SET state = 'locked',
@@ -1953,6 +2017,32 @@ func (q *Queries) NextDocumentVersionSeq(ctx context.Context, documentID ident.U
return seq, err return seq, err
} }
const purgeDocumentVersion = `-- name: PurgeDocumentVersion :execrows
UPDATE document_version
SET content_purged_at = ?1
WHERE document_version.id = ?2
AND document_version.content_purged_at IS NULL
AND NOT EXISTS (
SELECT 1
FROM document d
WHERE d.id = document_version.document_id
AND d.head_version_id = document_version.id
)
`
type PurgeDocumentVersionParams struct {
PurgedAt sql.NullInt64
ID ident.UUID
}
func (q *Queries) PurgeDocumentVersion(ctx context.Context, arg PurgeDocumentVersionParams) (int64, error) {
result, err := q.db.ExecContext(ctx, purgeDocumentVersion, arg.PurgedAt, arg.ID)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
const recordLogin = `-- name: RecordLogin :one const recordLogin = `-- name: RecordLogin :one
UPDATE account UPDATE account
SET last_login_at = ?1, SET last_login_at = ?1,
@@ -2322,6 +2412,49 @@ func (q *Queries) UpdateAccountState(ctx context.Context, arg UpdateAccountState
return i, err return i, err
} }
const updateBlobLocationState = `-- name: UpdateBlobLocationState :exec
UPDATE blob_location
SET state = ?1
WHERE root = ?2
AND backend_id = ?3
`
type UpdateBlobLocationStateParams struct {
State string
Root []byte
BackendID ident.UUID
}
func (q *Queries) UpdateBlobLocationState(ctx context.Context, arg UpdateBlobLocationStateParams) error {
_, err := q.db.ExecContext(ctx, updateBlobLocationState, arg.State, arg.Root, arg.BackendID)
return err
}
const updateBlobVerification = `-- name: UpdateBlobVerification :exec
UPDATE blob
SET state = ?1,
last_verified_at = ?2,
verify_error = ?3
WHERE root = ?4
`
type UpdateBlobVerificationParams struct {
State string
LastVerifiedAt sql.NullInt64
VerifyError sql.NullString
Root []byte
}
func (q *Queries) UpdateBlobVerification(ctx context.Context, arg UpdateBlobVerificationParams) error {
_, err := q.db.ExecContext(ctx, updateBlobVerification,
arg.State,
arg.LastVerifiedAt,
arg.VerifyError,
arg.Root,
)
return err
}
const updateDocumentLocation = `-- name: UpdateDocumentLocation :exec const updateDocumentLocation = `-- name: UpdateDocumentLocation :exec
UPDATE document UPDATE document
SET path = ?1, SET path = ?1,
+46
View File
@@ -431,6 +431,40 @@ FROM blob_location
WHERE root = sqlc.arg(root) WHERE root = sqlc.arg(root)
ORDER BY created_at; ORDER BY created_at;
-- name: ListUnreferencedBlobs :many
SELECT b.*
FROM blob b
WHERE NOT EXISTS (
SELECT 1
FROM document_version v
WHERE v.content_root = b.root
AND v.content_purged_at IS NULL
)
AND NOT EXISTS (
SELECT 1
FROM export_link e
WHERE e.blob_root = b.root
AND e.state <> 'stale'
)
ORDER BY b.created_at;
-- name: DeleteBlob :exec
DELETE FROM blob
WHERE root = sqlc.arg(root);
-- name: UpdateBlobVerification :exec
UPDATE blob
SET state = sqlc.arg(state),
last_verified_at = sqlc.arg(last_verified_at),
verify_error = sqlc.narg(verify_error)
WHERE root = sqlc.arg(root);
-- name: UpdateBlobLocationState :exec
UPDATE blob_location
SET state = sqlc.arg(state)
WHERE root = sqlc.arg(root)
AND backend_id = sqlc.arg(backend_id);
-- name: CreateBlobLocation :exec -- name: CreateBlobLocation :exec
INSERT INTO blob_location ( INSERT INTO blob_location (
root, root,
@@ -680,6 +714,18 @@ SET head_version_id = sqlc.arg(head_version_id),
WHERE id = sqlc.arg(id) WHERE id = sqlc.arg(id)
AND deleted_at IS NULL; AND deleted_at IS NULL;
-- name: PurgeDocumentVersion :execrows
UPDATE document_version
SET content_purged_at = sqlc.arg(purged_at)
WHERE document_version.id = sqlc.arg(id)
AND document_version.content_purged_at IS NULL
AND NOT EXISTS (
SELECT 1
FROM document d
WHERE d.id = document_version.document_id
AND d.head_version_id = document_version.id
);
-- ============================================================================ -- ============================================================================
-- acl_entry -- acl_entry
-- ============================================================================ -- ============================================================================
+31
View File
@@ -28,6 +28,7 @@ type Staged struct {
type Store interface { type Store interface {
Stage(ctx context.Context, id ident.UUID, src io.Reader) (Staged, error) Stage(ctx context.Context, id ident.UUID, src io.Reader) (Staged, error)
Commit(ctx context.Context, staged Staged) (string, error) Commit(ctx context.Context, staged Staged) (string, error)
Replace(ctx context.Context, staged Staged) (string, error)
Open(ctx context.Context, locator string) (io.ReadCloser, error) Open(ctx context.Context, locator string) (io.ReadCloser, error)
Remove(ctx context.Context, locator string) error Remove(ctx context.Context, locator string) error
} }
@@ -136,6 +137,36 @@ func (l *Local) Commit(ctx context.Context, staged Staged) (string, error) {
return locator, nil return locator, nil
} }
func (l *Local) Replace(ctx context.Context, staged Staged) (string, error) {
if err := ctx.Err(); err != nil {
return "", err
}
if len(staged.Root) != sha256.Size {
return "", fmt.Errorf("%w: empreinte de %d octets", ErrInvalidLocator, len(staged.Root))
}
source, err := l.resolve(staged.Locator)
if err != nil {
return "", err
}
root := hex.EncodeToString(staged.Root)
locator := filepath.ToSlash(filepath.Join("blobs", root[:2], root[2:4], root))
target, err := l.resolve(locator)
if err != nil {
return "", err
}
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return "", err
}
if err := os.Rename(source, target); err != nil {
return "", err
}
return locator, nil
}
func (l *Local) Open(ctx context.Context, locator string) (io.ReadCloser, error) { func (l *Local) Open(ctx context.Context, locator string) (io.ReadCloser, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, err return nil, err
+52
View File
@@ -94,6 +94,58 @@ func TestLocalStageCommitDedupOpenRemove(t *testing.T) {
} }
} }
func TestLocalReplace(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
store, err := NewLocal(root)
if err != nil {
t.Fatalf("NewLocal : %v", err)
}
payload := []byte("contenu à réparer")
staged, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload))
if err != nil {
t.Fatalf("Stage : %v", err)
}
locator, err := store.Commit(ctx, staged)
if err != nil {
t.Fatalf("Commit : %v", err)
}
if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(locator)), []byte("corrompu"), 0o640); err != nil {
t.Fatalf("WriteFile : %v", err)
}
replacement, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload))
if err != nil {
t.Fatalf("Stage(remplacement) : %v", err)
}
replacedLocator, err := store.Replace(ctx, replacement)
if err != nil {
t.Fatalf("Replace : %v", err)
}
if replacedLocator != locator {
t.Fatalf("locator = %q, attendu %q", replacedLocator, locator)
}
reader, err := store.Open(ctx, locator)
if err != nil {
t.Fatalf("Open : %v", err)
}
data, err := io.ReadAll(reader)
if err != nil {
_ = reader.Close()
t.Fatalf("ReadAll : %v", err)
}
if err := reader.Close(); err != nil {
t.Fatalf("Close : %v", err)
}
if !bytes.Equal(data, payload) {
t.Fatalf("contenu = %q, attendu %q", data, payload)
}
}
func TestLocalRejectsInvalidLocators(t *testing.T) { func TestLocalRejectsInvalidLocators(t *testing.T) {
ctx := context.Background() ctx := context.Background()
store, err := NewLocal(t.TempDir()) store, err := NewLocal(t.TempDir())