Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generate thumbnails

Browse Source
This commit is contained in:
m
2026-07-16 07:35:08 +02:00
parent 79c743722d
commit 10c07d2946
16 changed files with 608 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Dockerfile
+2 -2
View File
@@ -11,14 +11,14 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/server ./cmd/server
FROM alpine:3.21
RUN apk add --no-cache ca-certificates curl
RUN apk add --no-cache ca-certificates curl libreoffice-core poppler-utils
WORKDIR /app
COPY --from=builder /bin/server .
COPY internal/db/migrations ./internal/db/migrations
RUN mkdir -p /app/uploads /data
RUN mkdir -p /app/uploads /app/uploads/thumbnails /data
EXPOSE 8080
backend/cmd/server/main.go
+5 -1
View File
@@ -32,6 +32,7 @@ func main() {
fileSvc := service.NewFileService(queries, cfg)
ocrSvc := service.NewOCRService(cfg, fileSvc)
conversionSvc := service.NewConversionService(queries, cfg)
urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost)
authSvc, err := auth.NewAuthService(queries, cfg)
if err != nil {
@@ -41,7 +42,10 @@ func main() {
ocrSvc.Start()
defer ocrSvc.Stop()
h := handler.New(fileSvc, ocrSvc, urlSvc, auth.NewAuthHandler(authSvc))
conversionSvc.Start()
defer conversionSvc.Stop()
h := handler.New(fileSvc, ocrSvc, urlSvc, auth.NewAuthHandler(authSvc), conversionSvc)
r := gin.Default()
handler.SetupRoutes(r, h, authSvc)
backend/internal/config/config.go
+6
View File
@@ -10,6 +10,9 @@ type Config struct {
HMACSecret string
ServerHost string
PASETOKey string
LibreOfficePath string
PdftoppmPath string
ThumbnailDir string
}
func Load() *Config {
@@ -21,6 +24,9 @@ func Load() *Config {
HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"),
ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"),
PASETOKey: envOr("PASETO_KEY", "01234567890123456789012345678901234567890123456789012345678901234"),
LibreOfficePath: envOr("LIBREOFFICE_PATH", "/usr/bin/libreoffice"),
PdftoppmPath: envOr("PDFTOPPM_PATH", "/usr/bin/pdftoppm"),
ThumbnailDir: envOr("THUMBNAIL_DIR", "./uploads/thumbnails"),
}
}
backend/internal/db/migrations/011_thumbnails.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS thumbnails;
backend/internal/db/migrations/011_thumbnails.up.sql
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE thumbnails (
id TEXT PRIMARY KEY DEFAULT gen_random_uuid(),
file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE,
page_number INTEGER NOT NULL,
resolution_label TEXT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
storage_key TEXT NOT NULL,
mime_type TEXT NOT NULL DEFAULT 'image/jpeg',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX idx_thumbnails_file_id ON thumbnails(file_id);
CREATE UNIQUE INDEX idx_thumbnails_unique ON thumbnails(file_id, page_number, resolution_label);
backend/internal/db/models.go
+12
View File
@@ -47,6 +47,18 @@ type Tag struct {
UpdatedAt time.Time `json:"updated_at"`
}
type Thumbnail struct {
ID string `json:"id"`
FileID string `json:"file_id"`
PageNumber int32 `json:"page_number"`
ResolutionLabel string `json:"resolution_label"`
Width int32 `json:"width"`
Height int32 `json:"height"`
StorageKey string `json:"storage_key"`
MimeType string `json:"mime_type"`
CreatedAt time.Time `json:"created_at"`
}
type User struct {
ID string `json:"id"`
Username string `json:"username"`
backend/internal/db/queries/thumbnails.sql
+21
View File
@@ -0,0 +1,21 @@
-- name: CreateThumbnail :one
INSERT INTO thumbnails (file_id, page_number, resolution_label, width, height, storage_key, mime_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *;
-- name: GetThumbnailsByFileID :many
SELECT * FROM thumbnails
WHERE file_id = $1
ORDER BY page_number ASC, resolution_label ASC;
-- name: GetThumbnailByID :one
SELECT * FROM thumbnails
WHERE id = $1 LIMIT 1;
-- name: DeleteThumbnailsByFileID :exec
DELETE FROM thumbnails WHERE file_id = $1;
-- name: GetThumbnailByFilePageResolution :one
SELECT * FROM thumbnails
WHERE file_id = $1 AND page_number = $2 AND resolution_label = $3
LIMIT 1;
backend/internal/db/thumbnails.sql.go
+150
View File
@@ -0,0 +1,150 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: thumbnails.sql
package db
import (
"context"
)
const createThumbnail = `-- name: CreateThumbnail :one
INSERT INTO thumbnails (file_id, page_number, resolution_label, width, height, storage_key, mime_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id, file_id, page_number, resolution_label, width, height, storage_key, mime_type, created_at
`
type CreateThumbnailParams struct {
FileID string `json:"file_id"`
PageNumber int32 `json:"page_number"`
ResolutionLabel string `json:"resolution_label"`
Width int32 `json:"width"`
Height int32 `json:"height"`
StorageKey string `json:"storage_key"`
MimeType string `json:"mime_type"`
}
func (q *Queries) CreateThumbnail(ctx context.Context, arg CreateThumbnailParams) (Thumbnail, error) {
row := q.db.QueryRowContext(ctx, createThumbnail,
arg.FileID,
arg.PageNumber,
arg.ResolutionLabel,
arg.Width,
arg.Height,
arg.StorageKey,
arg.MimeType,
)
var i Thumbnail
err := row.Scan(
&i.ID,
&i.FileID,
&i.PageNumber,
&i.ResolutionLabel,
&i.Width,
&i.Height,
&i.StorageKey,
&i.MimeType,
&i.CreatedAt,
)
return i, err
}
const deleteThumbnailsByFileID = `-- name: DeleteThumbnailsByFileID :exec
DELETE FROM thumbnails WHERE file_id = $1
`
func (q *Queries) DeleteThumbnailsByFileID(ctx context.Context, fileID string) error {
_, err := q.db.ExecContext(ctx, deleteThumbnailsByFileID, fileID)
return err
}
const getThumbnailByFilePageResolution = `-- name: GetThumbnailByFilePageResolution :one
SELECT id, file_id, page_number, resolution_label, width, height, storage_key, mime_type, created_at FROM thumbnails
WHERE file_id = $1 AND page_number = $2 AND resolution_label = $3
LIMIT 1
`
type GetThumbnailByFilePageResolutionParams struct {
FileID string `json:"file_id"`
PageNumber int32 `json:"page_number"`
ResolutionLabel string `json:"resolution_label"`
}
func (q *Queries) GetThumbnailByFilePageResolution(ctx context.Context, arg GetThumbnailByFilePageResolutionParams) (Thumbnail, error) {
row := q.db.QueryRowContext(ctx, getThumbnailByFilePageResolution, arg.FileID, arg.PageNumber, arg.ResolutionLabel)
var i Thumbnail
err := row.Scan(
&i.ID,
&i.FileID,
&i.PageNumber,
&i.ResolutionLabel,
&i.Width,
&i.Height,
&i.StorageKey,
&i.MimeType,
&i.CreatedAt,
)
return i, err
}
const getThumbnailByID = `-- name: GetThumbnailByID :one
SELECT id, file_id, page_number, resolution_label, width, height, storage_key, mime_type, created_at FROM thumbnails
WHERE id = $1 LIMIT 1
`
func (q *Queries) GetThumbnailByID(ctx context.Context, id string) (Thumbnail, error) {
row := q.db.QueryRowContext(ctx, getThumbnailByID, id)
var i Thumbnail
err := row.Scan(
&i.ID,
&i.FileID,
&i.PageNumber,
&i.ResolutionLabel,
&i.Width,
&i.Height,
&i.StorageKey,
&i.MimeType,
&i.CreatedAt,
)
return i, err
}
const getThumbnailsByFileID = `-- name: GetThumbnailsByFileID :many
SELECT id, file_id, page_number, resolution_label, width, height, storage_key, mime_type, created_at FROM thumbnails
WHERE file_id = $1
ORDER BY page_number ASC, resolution_label ASC
`
func (q *Queries) GetThumbnailsByFileID(ctx context.Context, fileID string) ([]Thumbnail, error) {
rows, err := q.db.QueryContext(ctx, getThumbnailsByFileID, fileID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Thumbnail
for rows.Next() {
var i Thumbnail
if err := rows.Scan(
&i.ID,
&i.FileID,
&i.PageNumber,
&i.ResolutionLabel,
&i.Width,
&i.Height,
&i.StorageKey,
&i.MimeType,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
backend/internal/handler/files.go
+58
View File
@@ -15,6 +15,7 @@ type FileHandler struct {
files *service.FileService
urls *service.URLService
ocr *service.OCRService
conversion *service.ConversionService
}
func (h *FileHandler) Upload(c *gin.Context) {
@@ -40,6 +41,10 @@ func (h *FileHandler) Upload(c *gin.Context) {
h.ocr.Enqueue(result.ID, result.Path)
if service.IsConvertible(result.MimeType) {
h.conversion.Enqueue(result.ID, result.Path, result.MimeType)
}
results = append(results, gin.H{
"id": result.ID,
"name": result.Name,
@@ -252,3 +257,56 @@ func (h *FileHandler) ListFilesByParent(c *gin.Context) {
}
api.Success(c, files)
}
func (h *FileHandler) GetThumbnails(c *gin.Context) {
id := c.Param("id")
thumbnails, err := h.files.GetThumbnailsByFileID(id)
if err != nil {
api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to fetch thumbnails")
return
}
type thumbnailResponse struct {
ID string `json:"id"`
PageNumber int `json:"pageNumber"`
ResolutionLabel string `json:"resolutionLabel"`
Width int `json:"width"`
Height int `json:"height"`
URL string `json:"url"`
MimeType string `json:"mimeType"`
}
resp := make([]thumbnailResponse, len(thumbnails))
for i, t := range thumbnails {
resp[i] = thumbnailResponse{
ID: t.ID,
PageNumber: t.PageNumber,
ResolutionLabel: t.ResolutionLabel,
Width: t.Width,
Height: t.Height,
URL: h.urls.GenerateThumbnailURL(t.ID),
MimeType: t.MimeType,
}
}
api.Success(c, resp)
}
func (h *FileHandler) ServeThumbnail(c *gin.Context) {
id := c.Param("id")
exp, _ := strconv.ParseInt(c.Query("expires"), 10, 64)
sig := c.Query("sig")
if !h.urls.Validate(id, sig, exp) {
api.Error(c, http.StatusForbidden, "FORBIDDEN", "Invalid or expired link")
return
}
storagePath, err := h.files.GetThumbnailStoragePath(id)
if err != nil {
api.Error(c, http.StatusNotFound, "THUMBNAIL_NOT_FOUND", "Thumbnail not found")
return
}
c.File(path.Clean(storagePath))
}
backend/internal/handler/handler.go
+2 -2
View File
@@ -12,9 +12,9 @@ type Handler struct {
Auth *auth.AuthHandler
}
func New(fileSvc *service.FileService, ocrSvc *service.OCRService, urlSvc *service.URLService, authHandler *auth.AuthHandler) *Handler {
func New(fileSvc *service.FileService, ocrSvc *service.OCRService, urlSvc *service.URLService, authHandler *auth.AuthHandler, conversionSvc *service.ConversionService) *Handler {
return &Handler{
File: &FileHandler{files: fileSvc, urls: urlSvc, ocr: ocrSvc},
File: &FileHandler{files: fileSvc, urls: urlSvc, ocr: ocrSvc, conversion: conversionSvc},
OCR: &OCRHandler{ocr: ocrSvc, files: fileSvc},
Health: &HealthHandler{ocr: ocrSvc},
Auth: authHandler,
backend/internal/handler/router.go
+4
View File
@@ -34,4 +34,8 @@ func SetupRoutes(r *gin.Engine, h *Handler, authMiddleware *auth.AuthService) {
protected.POST("/ocr/jobs", h.OCR.CreateJob)
protected.GET("/ocr/jobs/:id", h.OCR.GetJobStatus)
protected.GET("/files/:id/thumbnails", h.File.GetThumbnails)
api.GET("/thumbnails/:id", h.File.ServeThumbnail)
}
backend/internal/model/file.go
+1
View File
@@ -25,4 +25,5 @@ type UploadResult struct {
ID string `json:"id"`
Name string `json:"name"`
Path string `json:"path"`
MimeType string `json:"mimeType"`
}
backend/internal/model/thumbnail.go
+13
View File
@@ -0,0 +1,13 @@
package model
type Thumbnail struct {
ID string `json:"id"`
FileID string `json:"fileId"`
PageNumber int `json:"pageNumber"`
ResolutionLabel string `json:"resolutionLabel"`
Width int `json:"width"`
Height int `json:"height"`
StorageKey string `json:"-"`
MimeType string `json:"mimeType"`
CreatedAt string `json:"createdAt"`
}
backend/internal/service/conversion.go
+251
View File
@@ -0,0 +1,251 @@
package service
import (
"context"
"fmt"
"image"
_ "image/jpeg"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/google/uuid"
"github.com/vaultdrop/backend/internal/config"
"github.com/vaultdrop/backend/internal/db"
)
type ConversionJob struct {
FileID string
FilePath string
MimeType string
}
type ConversionService struct {
queries *db.Queries
cfg *config.Config
jobs chan ConversionJob
}
func NewConversionService(queries *db.Queries, cfg *config.Config) *ConversionService {
return &ConversionService{
queries: queries,
cfg: cfg,
jobs: make(chan ConversionJob, 100),
}
}
func (s *ConversionService) Start() {
go s.worker()
log.Println("[Conversion] Worker started")
}
func (s *ConversionService) Stop() {
close(s.jobs)
log.Println("[Conversion] Worker stopped")
}
func (s *ConversionService) Enqueue(fileID, filePath, mimeType string) {
s.jobs <- ConversionJob{FileID: fileID, FilePath: filePath, MimeType: mimeType}
log.Printf("[Conversion] Enqueued file %s", fileID)
}
func (s *ConversionService) worker() {
for job := range s.jobs {
s.process(job)
}
}
func (s *ConversionService) process(job ConversionJob) {
log.Printf("[Conversion] Processing file %s (mime: %s)", job.FileID, job.MimeType)
pdfPath := job.FilePath
tmpDir := ""
if isOfficeDocument(job.MimeType) {
var err error
pdfPath, tmpDir, err = s.convertToPDF(job.FilePath)
if err != nil {
log.Printf("[Conversion] Failed to convert file %s to PDF: %v", job.FileID, err)
return
}
defer os.RemoveAll(tmpDir)
} else if !isPDF(job.MimeType) {
log.Printf("[Conversion] Skipping file %s: unsupported mime type %s", job.FileID, job.MimeType)
return
}
thumbDir := filepath.Join(s.cfg.ThumbnailDir, job.FileID)
if err := os.MkdirAll(thumbDir, 0o755); err != nil {
log.Printf("[Conversion] Failed to create thumbnail dir for %s: %v", job.FileID, err)
return
}
resolutions := []struct {
label string
dpi int
}{
{"thumbnail", 21},
{"full", 200},
}
for _, res := range resolutions {
pages, err := s.convertPDFToImages(pdfPath, thumbDir, res.dpi)
if err != nil {
log.Printf("[Conversion] Failed to convert file %s to images (res=%s): %v", job.FileID, res.label, err)
continue
}
for _, page := range pages {
width, height, err := getImageDimensions(page.path)
if err != nil {
log.Printf("[Conversion] Failed to get dimensions for %s: %v", page.path, err)
width, height = 0, 0
}
thumbUUID := uuid.New().String()
dstPath := filepath.Join(thumbDir, thumbUUID+".jpg")
if err := os.Rename(page.path, dstPath); err != nil {
log.Printf("[Conversion] Failed to move %s to %s: %v", page.path, dstPath, err)
continue
}
_, err = s.queries.CreateThumbnail(context.Background(), db.CreateThumbnailParams{
FileID: job.FileID,
PageNumber: int32(page.number),
ResolutionLabel: res.label,
Width: int32(width),
Height: int32(height),
StorageKey: dstPath,
MimeType: "image/jpeg",
})
if err != nil {
log.Printf("[Conversion] Failed to create thumbnail record for file %s page %d: %v", job.FileID, page.number, err)
continue
}
}
log.Printf("[Conversion] Generated %d %s images for file %s", len(pages), res.label, job.FileID)
}
log.Printf("[Conversion] Completed file %s", job.FileID)
}
func (s *ConversionService) convertToPDF(inputPath string) (string, string, error) {
tmpDir, err := os.MkdirTemp("", "conversion-*")
if err != nil {
return "", "", fmt.Errorf("create temp dir: %w", err)
}
cmd := exec.Command(s.cfg.LibreOfficePath,
"--headless",
"--convert-to", "pdf",
"--outdir", tmpDir,
inputPath,
)
output, err := cmd.CombinedOutput()
if err != nil {
os.RemoveAll(tmpDir)
return "", "", fmt.Errorf("libreoffice conversion failed: %s: %w", string(output), err)
}
baseName := filepath.Base(inputPath)
pdfName := strings.TrimSuffix(baseName, filepath.Ext(baseName)) + ".pdf"
pdfPath := filepath.Join(tmpDir, pdfName)
if _, err := os.Stat(pdfPath); os.IsNotExist(err) {
os.RemoveAll(tmpDir)
return "", "", fmt.Errorf("PDF not found at %s", pdfPath)
}
return pdfPath, tmpDir, nil
}
type imagePage struct {
number int
path string
}
func (s *ConversionService) convertPDFToImages(pdfPath, outputDir string, dpi int) ([]imagePage, error) {
prefix := filepath.Join(outputDir, fmt.Sprintf("tmp_%d_", dpi))
cmd := exec.Command(s.cfg.PdftoppmPath,
"-jpeg",
"-r", fmt.Sprintf("%d", dpi),
pdfPath,
prefix,
)
output, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("pdftoppm failed: %s: %w", string(output), err)
}
entries, err := os.ReadDir(outputDir)
if err != nil {
return nil, fmt.Errorf("read output dir: %w", err)
}
var pages []imagePage
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasPrefix(name, fmt.Sprintf("tmp_%d_", dpi)) || !strings.HasSuffix(name, ".jpg") {
continue
}
var num int
if _, err := fmt.Sscanf(strings.TrimPrefix(name, fmt.Sprintf("tmp_%d_", dpi)), "%d", &num); err != nil {
continue
}
pages = append(pages, imagePage{
number: num,
path: filepath.Join(outputDir, name),
})
}
return pages, nil
}
func isPDF(mimeType string) bool {
return strings.Contains(mimeType, "pdf")
}
func isOfficeDocument(mimeType string) bool {
officeTypes := []string{
"application/vnd.openxmlformats-officedocument",
"application/vnd.ms-excel",
"application/msword",
"application/vnd.ms-powerpoint",
"application/vnd.oasis.opendocument",
"application/x-doc",
"application/x-xls",
"application/x-ppt",
}
for _, t := range officeTypes {
if strings.Contains(mimeType, t) {
return true
}
}
return false
}
func IsConvertible(mimeType string) bool {
return isPDF(mimeType) || isOfficeDocument(mimeType)
}
func getImageDimensions(path string) (int, int, error) {
f, err := os.Open(path)
if err != nil {
return 0, 0, err
}
defer f.Close()
cfg, _, err := image.DecodeConfig(f)
if err != nil {
return 0, 0, err
}
return cfg.Width, cfg.Height, nil
}
backend/internal/service/file.go
+32
View File
@@ -62,6 +62,7 @@ func (s *FileService) Upload(file *multipart.FileHeader) (*model.UploadResult, e
ID: dbFile.ID,
Name: dbFile.Name,
Path: dst,
MimeType: dbFile.MimeType,
}, nil
}
@@ -206,6 +207,37 @@ func (s *FileService) ListFilesByParentID(parentID string) ([]model.File, error)
return files, nil
}
func (s *FileService) GetThumbnailsByFileID(fileID string) ([]model.Thumbnail, error) {
dbThumbnails, err := s.queries.GetThumbnailsByFileID(context.Background(), fileID)
if err != nil {
return nil, fmt.Errorf("get thumbnails: %w", err)
}
thumbnails := make([]model.Thumbnail, len(dbThumbnails))
for i, t := range dbThumbnails {
thumbnails[i] = model.Thumbnail{
ID: t.ID,
FileID: t.FileID,
PageNumber: int(t.PageNumber),
ResolutionLabel: t.ResolutionLabel,
Width: int(t.Width),
Height: int(t.Height),
StorageKey: t.StorageKey,
MimeType: t.MimeType,
CreatedAt: t.CreatedAt.String(),
}
}
return thumbnails, nil
}
func (s *FileService) GetThumbnailStoragePath(id string) (string, error) {
t, err := s.queries.GetThumbnailByID(context.Background(), id)
if err != nil {
return "", fmt.Errorf("get thumbnail: %w", err)
}
return t.StorageKey, nil
}
func dbToModel(f db.File, dbTags []db.Tag) model.File {
tags := make([]model.Tag, len(dbTags))
for i, t := range dbTags {
backend/internal/service/url.go
+13
View File
@@ -37,6 +37,19 @@ func (s *URLService) GenerateDownloadURL(fileUUID string) string {
)
}
func (s *URLService) GenerateThumbnailURL(thumbUUID string) string {
expires := time.Now().Add(10 * time.Minute).Unix()
sig := s.sign(thumbUUID, expires)
return fmt.Sprintf(
"%s/api/v1/thumbnails/%s?expires=%d&sig=%s",
s.serverHost,
thumbUUID,
expires,
sig,
)
}
func (s *URLService) Validate(fileID, sig string, expires int64) bool {
if time.Now().Unix() > expires {
return false