Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

display files with cache image

Browse Source
This commit is contained in:
m
2026-07-27 23:02:09 +02:00
parent c5a959a27b
commit e8ceb99a5d
25 changed files with 814 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/cmd/gc/main.go
+147
View File
@@ -0,0 +1,147 @@
package main
import (
"database/sql"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/joho/godotenv"
_ "github.com/lib/pq"
"github.com/vaultdrop/backend/internal/config"
)
func main() {
_ = godotenv.Load()
cfg := config.Load()
db, err := sql.Open("postgres", cfg.DatabaseURL)
if err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}
defer db.Close()
uploadDir := cfg.UploadDir
thumbnailDir := cfg.ThumbnailDir
fmt.Println("=== VaultDrop Orphan GC ===")
fmt.Printf("Upload dir: %s\n", uploadDir)
fmt.Printf("Thumbnail dir: %s\n", thumbnailDir)
var fileCount int
err = db.QueryRow("SELECT COUNT(*) FROM files WHERE is_folder = false AND storage_key != ''").Scan(&fileCount)
if err != nil {
log.Fatalf("Failed to count files: %v", err)
}
fmt.Printf("Files in DB: %d\n", fileCount)
rows, err := db.Query("SELECT id, storage_key FROM files WHERE is_folder = false AND storage_key != ''")
if err != nil {
log.Fatalf("Failed to query files: %v", err)
}
defer rows.Close()
dbPaths := make(map[string]string)
for rows.Next() {
var id, storageKey string
if err := rows.Scan(&id, &storageKey); err != nil {
continue
}
dbPaths[storageKey] = id
}
entries, err := os.ReadDir(uploadDir)
if err != nil {
log.Fatalf("Failed to read upload dir: %v", err)
}
orphanFiles := 0
freedBytes := int64(0)
for _, entry := range entries {
if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
continue
}
fullPath := filepath.Join(uploadDir, entry.Name())
relPath := "./" + fullPath
if _, ok := dbPaths[fullPath]; !ok {
if _, ok2 := dbPaths[relPath]; !ok2 {
info, err := entry.Info()
if err == nil {
freedBytes += info.Size()
}
orphanFiles++
fmt.Printf(" ORPHAN FILE: %s\n", fullPath)
os.Remove(fullPath)
}
}
}
var thumbCount int
err = db.QueryRow("SELECT COUNT(*) FROM thumbnails").Scan(&thumbCount)
if err != nil {
log.Fatalf("Failed to count thumbnails: %v", err)
}
fmt.Printf("Thumbnails in DB: %d\n", thumbCount)
thumbRows, err := db.Query("SELECT id, file_id, storage_key FROM thumbnails")
if err != nil {
log.Fatalf("Failed to query thumbnails: %v", err)
}
defer thumbRows.Close()
dbThumbPaths := make(map[string]string)
for thumbRows.Next() {
var id, fileID, storageKey string
if err := thumbRows.Scan(&id, &fileID, &storageKey); err != nil {
continue
}
dbThumbPaths[storageKey] = fileID
}
if _, err := os.Stat(thumbnailDir); err == nil {
fileDirs, err := os.ReadDir(thumbnailDir)
if err == nil {
for _, fileDir := range fileDirs {
if !fileDir.IsDir() {
continue
}
fileDirPath := filepath.Join(thumbnailDir, fileDir.Name())
thumbFiles, err := os.ReadDir(fileDirPath)
if err != nil {
continue
}
for _, tf := range thumbFiles {
if tf.IsDir() || strings.HasPrefix(tf.Name(), ".") {
continue
}
thumbPath := filepath.Join(fileDirPath, tf.Name())
relThumbPath := "./" + thumbPath
if _, ok := dbThumbPaths[thumbPath]; !ok {
if _, ok2 := dbThumbPaths[relThumbPath]; !ok2 {
info, err := tf.Info()
if err == nil {
freedBytes += info.Size()
}
orphanFiles++
fmt.Printf(" ORPHAN THUMB: %s\n", thumbPath)
os.Remove(thumbPath)
}
}
}
remaining, _ := os.ReadDir(fileDirPath)
if len(remaining) == 0 {
os.Remove(fileDirPath)
}
}
}
}
fmt.Printf("\n=== Summary ===\n")
fmt.Printf("Orphan files removed: %d\n", orphanFiles)
fmt.Printf("Space freed: %.2f MB\n", float64(freedBytes)/(1024*1024))
}
backend/cmd/server/main.go
+1 -1
View File
@@ -33,7 +33,7 @@ func main() {
fileSvc := service.NewFileService(queries, cfg)
ocrSvc := service.NewOCRService(cfg, fileSvc)
conversionSvc := service.NewConversionService(queries, cfg)
urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost)
urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost, cfg.URLExpiryMinutes)
authSvc, err := auth.NewAuthService(queries, cfg)
if err != nil {
log.Fatalf("Failed to create auth service: %v", err)
backend/internal/config/config.go
+36 -21
View File
@@ -1,35 +1,50 @@
package config
import "os"
import (
"fmt"
"os"
)
type Config struct {
Port string
DatabaseURL string
OCREndpoint string
UploadDir string
HMACSecret string
ServerHost string
PASETOKey string
LibreOfficePath string
PdftoppmPath string
ThumbnailDir string
Port string
DatabaseURL string
OCREndpoint string
UploadDir string
HMACSecret string
ServerHost string
PASETOKey string
LibreOfficePath string
PdftoppmPath string
ThumbnailDir string
URLExpiryMinutes int
}
func Load() *Config {
return &Config{
Port: envOr("PORT", "8080"),
DatabaseURL: envOr("DATABASE_URL", "postgres://localhost:5432/vaultdrop?sslmode=disable"),
OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"),
UploadDir: envOr("UPLOAD_DIR", "./uploads"),
HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"),
ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"),
PASETOKey: envOr("PASETO_KEY", "01234567890123456789012345678901234567890123456789012345678901234"),
LibreOfficePath: envOr("LIBREOFFICE_PATH", "/usr/bin/libreoffice"),
PdftoppmPath: envOr("PDFTOPPM_PATH", "/usr/bin/pdftoppm"),
ThumbnailDir: envOr("THUMBNAIL_DIR", "./uploads/thumbnails"),
Port: envOr("PORT", "8080"),
DatabaseURL: envOr("DATABASE_URL", "postgres://localhost:5432/vaultdrop?sslmode=disable"),
OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"),
UploadDir: envOr("UPLOAD_DIR", "./uploads"),
HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"),
ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"),
PASETOKey: envOr("PASETO_KEY", "01234567890123456789012345678901234567890123456789012345678901234"),
LibreOfficePath: envOr("LIBREOFFICE_PATH", "/usr/bin/libreoffice"),
PdftoppmPath: envOr("PDFTOPPM_PATH", "/usr/bin/pdftoppm"),
ThumbnailDir: envOr("THUMBNAIL_DIR", "./uploads/thumbnails"),
URLExpiryMinutes: envOrInt("URL_EXPIRY_MINUTES", 60),
}
}
func envOrInt(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
var n int
if _, err := fmt.Sscanf(v, "%d", &n); err == nil && n > 0 {
return n
}
}
return fallback
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
backend/internal/db/files.sql.go
+76
View File
@@ -8,6 +8,7 @@ package db
import (
"context"
"database/sql"
"time"
"github.com/lib/pq"
)
@@ -86,6 +87,81 @@ func (q *Queries) DeleteFile(ctx context.Context, id string) error {
return err
}
const findDuplicateByChecksum = `-- name: FindDuplicateByChecksum :one
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, parent_file_id, is_folder FROM files
WHERE checksum = $1 AND is_folder = false
LIMIT 1
`
func (q *Queries) FindDuplicateByChecksum(ctx context.Context, checksum string) (File, error) {
row := q.db.QueryRowContext(ctx, findDuplicateByChecksum, checksum)
var i File
err := row.Scan(
&i.ID,
&i.Name,
&i.MimeType,
&i.Size,
&i.StorageKey,
&i.Checksum,
&i.OcrText,
&i.CreatedAt,
&i.UpdatedAt,
&i.ParentFileID,
&i.IsFolder,
)
return i, err
}
const findDuplicatesByNameSize = `-- name: FindDuplicatesByNameSize :many
SELECT id, name, mime_type, size, checksum, created_at FROM files
WHERE name = $1 AND size = $2 AND is_folder = false
ORDER BY created_at DESC
`
type FindDuplicatesByNameSizeParams struct {
Name string `json:"name"`
Size int64 `json:"size"`
}
type FindDuplicatesByNameSizeRow struct {
ID string `json:"id"`
Name string `json:"name"`
MimeType string `json:"mime_type"`
Size int64 `json:"size"`
Checksum string `json:"checksum"`
CreatedAt time.Time `json:"created_at"`
}
func (q *Queries) FindDuplicatesByNameSize(ctx context.Context, arg FindDuplicatesByNameSizeParams) ([]FindDuplicatesByNameSizeRow, error) {
rows, err := q.db.QueryContext(ctx, findDuplicatesByNameSize, arg.Name, arg.Size)
if err != nil {
return nil, err
}
defer rows.Close()
var items []FindDuplicatesByNameSizeRow
for rows.Next() {
var i FindDuplicatesByNameSizeRow
if err := rows.Scan(
&i.ID,
&i.Name,
&i.MimeType,
&i.Size,
&i.Checksum,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getFile = `-- name: GetFile :one
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, parent_file_id, is_folder FROM files
WHERE id = $1 LIMIT 1
backend/internal/db/queries/files.sql
+10
View File
@@ -45,3 +45,13 @@ WHERE id = $4;
-- name: DeleteFile :exec
DELETE FROM files
WHERE id = $1;
-- name: FindDuplicatesByNameSize :many
SELECT id, name, mime_type, size, checksum, created_at FROM files
WHERE name = $1 AND size = $2 AND is_folder = false
ORDER BY created_at DESC;
-- name: FindDuplicateByChecksum :one
SELECT * FROM files
WHERE checksum = $1 AND is_folder = false
LIMIT 1;
backend/internal/handler/files.go
+57
View File
@@ -3,6 +3,7 @@ package handler
import (
"log"
"net/http"
"os"
"path"
"strconv"
@@ -218,10 +219,22 @@ func (h *FileHandler) Get(c *gin.Context) {
func (h *FileHandler) Delete(c *gin.Context) {
id := c.Param("id")
storagePath, _ := h.files.GetStoragePath(id)
thumbnails, _ := h.files.GetThumbnailsByFileID(id)
if err := h.files.Delete(id); err != nil {
api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to delete file")
return
}
if storagePath != "" {
os.Remove(path.Clean(storagePath))
}
for _, t := range thumbnails {
os.Remove(path.Clean(t.StorageKey))
}
api.Success(c, gin.H{"deleted": true})
}
@@ -432,3 +445,47 @@ func (h *FileHandler) ServeThumbnail(c *gin.Context) {
c.File(path.Clean(storagePath))
}
func (h *FileHandler) CheckDuplicates(c *gin.Context) {
var body struct {
Name string `json:"name" binding:"required"`
Size int64 `json:"size" binding:"required"`
MimeType string `json:"mime_type"`
}
if err := c.ShouldBindJSON(&body); err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_BODY", "Body must contain 'name' and 'size'")
return
}
duplicates, err := h.files.FindDuplicatesByNameSize(body.Name, body.Size)
if err != nil {
api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to check duplicates")
return
}
type dupResponse struct {
ID string `json:"id"`
Name string `json:"name"`
MimeType string `json:"mimeType"`
Size int64 `json:"size"`
Checksum string `json:"checksum"`
CreatedAt string `json:"createdAt"`
}
resp := make([]dupResponse, len(duplicates))
for i, d := range duplicates {
resp[i] = dupResponse{
ID: d.ID,
Name: d.Name,
MimeType: d.MimeType,
Size: d.Size,
Checksum: d.Checksum,
CreatedAt: d.CreatedAt.Format("2006-01-02 15:04:05"),
}
}
api.Success(c, gin.H{
"duplicates": resp,
"count": len(resp),
})
}
backend/internal/handler/router.go
+2
View File
@@ -37,4 +37,6 @@ func SetupRoutes(r *gin.Engine, h *Handler, authMiddleware *auth.AuthService) {
protected.GET("/ocr/jobs/:id", h.OCR.GetJobStatus)
protected.GET("/files/:id/thumbnails", h.File.GetThumbnails)
protected.POST("/files/dedup-check", h.File.CheckDuplicates)
}
backend/internal/service/file.go
+18
View File
@@ -47,6 +47,17 @@ func (s *FileService) Upload(file *multipart.FileHeader) (*model.UploadResult, e
checksum := hex.EncodeToString(CreateSHA256Hash(data))
existing, err := s.queries.FindDuplicateByChecksum(context.Background(), checksum)
if err == nil && existing.ID != "" {
os.Remove(dst)
return &model.UploadResult{
ID: existing.ID,
Name: existing.Name,
Path: existing.StorageKey,
MimeType: existing.MimeType,
}, nil
}
dbFile, err := s.queries.CreateFile(context.Background(), db.CreateFileParams{
Name: file.Filename,
MimeType: file.Header.Get("Content-Type"),
@@ -280,6 +291,13 @@ func (s *FileService) GetBestThumbnail(fileID, preferredLabel string) *model.Thu
return fallback
}
func (s *FileService) FindDuplicatesByNameSize(name string, size int64) ([]db.FindDuplicatesByNameSizeRow, error) {
return s.queries.FindDuplicatesByNameSize(context.Background(), db.FindDuplicatesByNameSizeParams{
Name: name,
Size: size,
})
}
func dbToModel(f db.File, dbTags []db.Tag) model.File {
tags := make([]model.Tag, len(dbTags))
for i, t := range dbTags {
backend/internal/service/url.go
+14 -6
View File
@@ -9,12 +9,20 @@ import (
)
type URLService struct {
secret string
serverHost string
secret string
serverHost string
expiryDuration time.Duration
}
func NewURLService(secret, serverHost string) *URLService {
return &URLService{secret: secret, serverHost: serverHost}
func NewURLService(secret, serverHost string, expiryMinutes int) *URLService {
if expiryMinutes <= 0 {
expiryMinutes = 60
}
return &URLService{
secret: secret,
serverHost: serverHost,
expiryDuration: time.Duration(expiryMinutes) * time.Minute,
}
}
func (s *URLService) sign(fileID string, expires int64) string {
@@ -25,7 +33,7 @@ func (s *URLService) sign(fileID string, expires int64) string {
}
func (s *URLService) GenerateDownloadURL(fileUUID string) string {
expires := time.Now().Add(10 * time.Minute).Unix()
expires := time.Now().Add(s.expiryDuration).Unix()
sig := s.sign(fileUUID, expires)
return fmt.Sprintf(
@@ -38,7 +46,7 @@ func (s *URLService) GenerateDownloadURL(fileUUID string) string {
}
func (s *URLService) GenerateThumbnailURL(thumbUUID string) string {
expires := time.Now().Add(10 * time.Minute).Unix()
expires := time.Now().Add(s.expiryDuration).Unix()
sig := s.sign(thumbUUID, expires)
return fmt.Sprintf(