diff --git a/backend/cmd/gc/main.go b/backend/cmd/gc/main.go new file mode 100644 index 0000000..3482499 --- /dev/null +++ b/backend/cmd/gc/main.go @@ -0,0 +1,147 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/joho/godotenv" + _ "github.com/lib/pq" + "github.com/vaultdrop/backend/internal/config" +) + +func main() { + _ = godotenv.Load() + + cfg := config.Load() + + db, err := sql.Open("postgres", cfg.DatabaseURL) + if err != nil { + log.Fatalf("Failed to connect to database: %v", err) + } + defer db.Close() + + uploadDir := cfg.UploadDir + thumbnailDir := cfg.ThumbnailDir + + fmt.Println("=== VaultDrop Orphan GC ===") + fmt.Printf("Upload dir: %s\n", uploadDir) + fmt.Printf("Thumbnail dir: %s\n", thumbnailDir) + + var fileCount int + err = db.QueryRow("SELECT COUNT(*) FROM files WHERE is_folder = false AND storage_key != ''").Scan(&fileCount) + if err != nil { + log.Fatalf("Failed to count files: %v", err) + } + fmt.Printf("Files in DB: %d\n", fileCount) + + rows, err := db.Query("SELECT id, storage_key FROM files WHERE is_folder = false AND storage_key != ''") + if err != nil { + log.Fatalf("Failed to query files: %v", err) + } + defer rows.Close() + + dbPaths := make(map[string]string) + for rows.Next() { + var id, storageKey string + if err := rows.Scan(&id, &storageKey); err != nil { + continue + } + dbPaths[storageKey] = id + } + + entries, err := os.ReadDir(uploadDir) + if err != nil { + log.Fatalf("Failed to read upload dir: %v", err) + } + + orphanFiles := 0 + freedBytes := int64(0) + for _, entry := range entries { + if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { + continue + } + fullPath := filepath.Join(uploadDir, entry.Name()) + relPath := "./" + fullPath + + if _, ok := dbPaths[fullPath]; !ok { + if _, ok2 := dbPaths[relPath]; !ok2 { + info, err := entry.Info() + if err == nil { + freedBytes += info.Size() + } + orphanFiles++ + fmt.Printf(" ORPHAN FILE: %s\n", fullPath) + os.Remove(fullPath) + } + } + } + + var thumbCount int + err = db.QueryRow("SELECT COUNT(*) FROM thumbnails").Scan(&thumbCount) + if err != nil { + log.Fatalf("Failed to count thumbnails: %v", err) + } + fmt.Printf("Thumbnails in DB: %d\n", thumbCount) + + thumbRows, err := db.Query("SELECT id, file_id, storage_key FROM thumbnails") + if err != nil { + log.Fatalf("Failed to query thumbnails: %v", err) + } + defer thumbRows.Close() + + dbThumbPaths := make(map[string]string) + for thumbRows.Next() { + var id, fileID, storageKey string + if err := thumbRows.Scan(&id, &fileID, &storageKey); err != nil { + continue + } + dbThumbPaths[storageKey] = fileID + } + + if _, err := os.Stat(thumbnailDir); err == nil { + fileDirs, err := os.ReadDir(thumbnailDir) + if err == nil { + for _, fileDir := range fileDirs { + if !fileDir.IsDir() { + continue + } + fileDirPath := filepath.Join(thumbnailDir, fileDir.Name()) + thumbFiles, err := os.ReadDir(fileDirPath) + if err != nil { + continue + } + for _, tf := range thumbFiles { + if tf.IsDir() || strings.HasPrefix(tf.Name(), ".") { + continue + } + thumbPath := filepath.Join(fileDirPath, tf.Name()) + relThumbPath := "./" + thumbPath + if _, ok := dbThumbPaths[thumbPath]; !ok { + if _, ok2 := dbThumbPaths[relThumbPath]; !ok2 { + info, err := tf.Info() + if err == nil { + freedBytes += info.Size() + } + orphanFiles++ + fmt.Printf(" ORPHAN THUMB: %s\n", thumbPath) + os.Remove(thumbPath) + } + } + } + + remaining, _ := os.ReadDir(fileDirPath) + if len(remaining) == 0 { + os.Remove(fileDirPath) + } + } + } + } + + fmt.Printf("\n=== Summary ===\n") + fmt.Printf("Orphan files removed: %d\n", orphanFiles) + fmt.Printf("Space freed: %.2f MB\n", float64(freedBytes)/(1024*1024)) +} diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 1b22714..ab1e0e7 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -33,7 +33,7 @@ func main() { fileSvc := service.NewFileService(queries, cfg) ocrSvc := service.NewOCRService(cfg, fileSvc) conversionSvc := service.NewConversionService(queries, cfg) - urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost) + urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost, cfg.URLExpiryMinutes) authSvc, err := auth.NewAuthService(queries, cfg) if err != nil { log.Fatalf("Failed to create auth service: %v", err) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 28ec35d..ef22da9 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -1,35 +1,50 @@ package config -import "os" +import ( + "fmt" + "os" +) type Config struct { - Port string - DatabaseURL string - OCREndpoint string - UploadDir string - HMACSecret string - ServerHost string - PASETOKey string - LibreOfficePath string - PdftoppmPath string - ThumbnailDir string + Port string + DatabaseURL string + OCREndpoint string + UploadDir string + HMACSecret string + ServerHost string + PASETOKey string + LibreOfficePath string + PdftoppmPath string + ThumbnailDir string + URLExpiryMinutes int } func Load() *Config { return &Config{ - Port: envOr("PORT", "8080"), - DatabaseURL: envOr("DATABASE_URL", "postgres://localhost:5432/vaultdrop?sslmode=disable"), - OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"), - UploadDir: envOr("UPLOAD_DIR", "./uploads"), - HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"), - ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"), - PASETOKey: envOr("PASETO_KEY", "01234567890123456789012345678901234567890123456789012345678901234"), - LibreOfficePath: envOr("LIBREOFFICE_PATH", "/usr/bin/libreoffice"), - PdftoppmPath: envOr("PDFTOPPM_PATH", "/usr/bin/pdftoppm"), - ThumbnailDir: envOr("THUMBNAIL_DIR", "./uploads/thumbnails"), + Port: envOr("PORT", "8080"), + DatabaseURL: envOr("DATABASE_URL", "postgres://localhost:5432/vaultdrop?sslmode=disable"), + OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"), + UploadDir: envOr("UPLOAD_DIR", "./uploads"), + HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"), + ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"), + PASETOKey: envOr("PASETO_KEY", "01234567890123456789012345678901234567890123456789012345678901234"), + LibreOfficePath: envOr("LIBREOFFICE_PATH", "/usr/bin/libreoffice"), + PdftoppmPath: envOr("PDFTOPPM_PATH", "/usr/bin/pdftoppm"), + ThumbnailDir: envOr("THUMBNAIL_DIR", "./uploads/thumbnails"), + URLExpiryMinutes: envOrInt("URL_EXPIRY_MINUTES", 60), } } +func envOrInt(key string, fallback int) int { + if v := os.Getenv(key); v != "" { + var n int + if _, err := fmt.Sscanf(v, "%d", &n); err == nil && n > 0 { + return n + } + } + return fallback +} + func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { return v diff --git a/backend/internal/db/files.sql.go b/backend/internal/db/files.sql.go index 1fd2ae2..b8f21d1 100644 --- a/backend/internal/db/files.sql.go +++ b/backend/internal/db/files.sql.go @@ -8,6 +8,7 @@ package db import ( "context" "database/sql" + "time" "github.com/lib/pq" ) @@ -86,6 +87,81 @@ func (q *Queries) DeleteFile(ctx context.Context, id string) error { return err } +const findDuplicateByChecksum = `-- name: FindDuplicateByChecksum :one +SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, parent_file_id, is_folder FROM files +WHERE checksum = $1 AND is_folder = false +LIMIT 1 +` + +func (q *Queries) FindDuplicateByChecksum(ctx context.Context, checksum string) (File, error) { + row := q.db.QueryRowContext(ctx, findDuplicateByChecksum, checksum) + var i File + err := row.Scan( + &i.ID, + &i.Name, + &i.MimeType, + &i.Size, + &i.StorageKey, + &i.Checksum, + &i.OcrText, + &i.CreatedAt, + &i.UpdatedAt, + &i.ParentFileID, + &i.IsFolder, + ) + return i, err +} + +const findDuplicatesByNameSize = `-- name: FindDuplicatesByNameSize :many +SELECT id, name, mime_type, size, checksum, created_at FROM files +WHERE name = $1 AND size = $2 AND is_folder = false +ORDER BY created_at DESC +` + +type FindDuplicatesByNameSizeParams struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +type FindDuplicatesByNameSizeRow struct { + ID string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mime_type"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + CreatedAt time.Time `json:"created_at"` +} + +func (q *Queries) FindDuplicatesByNameSize(ctx context.Context, arg FindDuplicatesByNameSizeParams) ([]FindDuplicatesByNameSizeRow, error) { + rows, err := q.db.QueryContext(ctx, findDuplicatesByNameSize, arg.Name, arg.Size) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FindDuplicatesByNameSizeRow + for rows.Next() { + var i FindDuplicatesByNameSizeRow + if err := rows.Scan( + &i.ID, + &i.Name, + &i.MimeType, + &i.Size, + &i.Checksum, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getFile = `-- name: GetFile :one SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, parent_file_id, is_folder FROM files WHERE id = $1 LIMIT 1 diff --git a/backend/internal/db/queries/files.sql b/backend/internal/db/queries/files.sql index 9e96868..da0b409 100644 --- a/backend/internal/db/queries/files.sql +++ b/backend/internal/db/queries/files.sql @@ -45,3 +45,13 @@ WHERE id = $4; -- name: DeleteFile :exec DELETE FROM files WHERE id = $1; + +-- name: FindDuplicatesByNameSize :many +SELECT id, name, mime_type, size, checksum, created_at FROM files +WHERE name = $1 AND size = $2 AND is_folder = false +ORDER BY created_at DESC; + +-- name: FindDuplicateByChecksum :one +SELECT * FROM files +WHERE checksum = $1 AND is_folder = false +LIMIT 1; diff --git a/backend/internal/handler/files.go b/backend/internal/handler/files.go index 9d8a9e0..db6d729 100644 --- a/backend/internal/handler/files.go +++ b/backend/internal/handler/files.go @@ -3,6 +3,7 @@ package handler import ( "log" "net/http" + "os" "path" "strconv" @@ -218,10 +219,22 @@ func (h *FileHandler) Get(c *gin.Context) { func (h *FileHandler) Delete(c *gin.Context) { id := c.Param("id") + + storagePath, _ := h.files.GetStoragePath(id) + thumbnails, _ := h.files.GetThumbnailsByFileID(id) + if err := h.files.Delete(id); err != nil { api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to delete file") return } + + if storagePath != "" { + os.Remove(path.Clean(storagePath)) + } + for _, t := range thumbnails { + os.Remove(path.Clean(t.StorageKey)) + } + api.Success(c, gin.H{"deleted": true}) } @@ -432,3 +445,47 @@ func (h *FileHandler) ServeThumbnail(c *gin.Context) { c.File(path.Clean(storagePath)) } + +func (h *FileHandler) CheckDuplicates(c *gin.Context) { + var body struct { + Name string `json:"name" binding:"required"` + Size int64 `json:"size" binding:"required"` + MimeType string `json:"mime_type"` + } + if err := c.ShouldBindJSON(&body); err != nil { + api.Error(c, http.StatusBadRequest, "INVALID_BODY", "Body must contain 'name' and 'size'") + return + } + + duplicates, err := h.files.FindDuplicatesByNameSize(body.Name, body.Size) + if err != nil { + api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to check duplicates") + return + } + + type dupResponse struct { + ID string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mimeType"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + CreatedAt string `json:"createdAt"` + } + + resp := make([]dupResponse, len(duplicates)) + for i, d := range duplicates { + resp[i] = dupResponse{ + ID: d.ID, + Name: d.Name, + MimeType: d.MimeType, + Size: d.Size, + Checksum: d.Checksum, + CreatedAt: d.CreatedAt.Format("2006-01-02 15:04:05"), + } + } + + api.Success(c, gin.H{ + "duplicates": resp, + "count": len(resp), + }) +} diff --git a/backend/internal/handler/router.go b/backend/internal/handler/router.go index 8025478..a5a2596 100644 --- a/backend/internal/handler/router.go +++ b/backend/internal/handler/router.go @@ -37,4 +37,6 @@ func SetupRoutes(r *gin.Engine, h *Handler, authMiddleware *auth.AuthService) { protected.GET("/ocr/jobs/:id", h.OCR.GetJobStatus) protected.GET("/files/:id/thumbnails", h.File.GetThumbnails) + + protected.POST("/files/dedup-check", h.File.CheckDuplicates) } diff --git a/backend/internal/service/file.go b/backend/internal/service/file.go index 8614826..d078599 100644 --- a/backend/internal/service/file.go +++ b/backend/internal/service/file.go @@ -47,6 +47,17 @@ func (s *FileService) Upload(file *multipart.FileHeader) (*model.UploadResult, e checksum := hex.EncodeToString(CreateSHA256Hash(data)) + existing, err := s.queries.FindDuplicateByChecksum(context.Background(), checksum) + if err == nil && existing.ID != "" { + os.Remove(dst) + return &model.UploadResult{ + ID: existing.ID, + Name: existing.Name, + Path: existing.StorageKey, + MimeType: existing.MimeType, + }, nil + } + dbFile, err := s.queries.CreateFile(context.Background(), db.CreateFileParams{ Name: file.Filename, MimeType: file.Header.Get("Content-Type"), @@ -280,6 +291,13 @@ func (s *FileService) GetBestThumbnail(fileID, preferredLabel string) *model.Thu return fallback } +func (s *FileService) FindDuplicatesByNameSize(name string, size int64) ([]db.FindDuplicatesByNameSizeRow, error) { + return s.queries.FindDuplicatesByNameSize(context.Background(), db.FindDuplicatesByNameSizeParams{ + Name: name, + Size: size, + }) +} + func dbToModel(f db.File, dbTags []db.Tag) model.File { tags := make([]model.Tag, len(dbTags)) for i, t := range dbTags { diff --git a/backend/internal/service/url.go b/backend/internal/service/url.go index d4b503f..5c55db3 100644 --- a/backend/internal/service/url.go +++ b/backend/internal/service/url.go @@ -9,12 +9,20 @@ import ( ) type URLService struct { - secret string - serverHost string + secret string + serverHost string + expiryDuration time.Duration } -func NewURLService(secret, serverHost string) *URLService { - return &URLService{secret: secret, serverHost: serverHost} +func NewURLService(secret, serverHost string, expiryMinutes int) *URLService { + if expiryMinutes <= 0 { + expiryMinutes = 60 + } + return &URLService{ + secret: secret, + serverHost: serverHost, + expiryDuration: time.Duration(expiryMinutes) * time.Minute, + } } func (s *URLService) sign(fileID string, expires int64) string { @@ -25,7 +33,7 @@ func (s *URLService) sign(fileID string, expires int64) string { } func (s *URLService) GenerateDownloadURL(fileUUID string) string { - expires := time.Now().Add(10 * time.Minute).Unix() + expires := time.Now().Add(s.expiryDuration).Unix() sig := s.sign(fileUUID, expires) return fmt.Sprintf( @@ -38,7 +46,7 @@ func (s *URLService) GenerateDownloadURL(fileUUID string) string { } func (s *URLService) GenerateThumbnailURL(thumbUUID string) string { - expires := time.Now().Add(10 * time.Minute).Unix() + expires := time.Now().Add(s.expiryDuration).Unix() sig := s.sign(thumbUUID, expires) return fmt.Sprintf( diff --git a/mobile/app.json b/mobile/app.json index a5ab6d8..e4666b8 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -44,7 +44,8 @@ "photosPermission": "VaultDrop a besoin d'accéder à vos photos pour les afficher et les organiser.", "videosPermission": "VaultDrop a besoin d'accéder à vos vidéos pour les afficher et les organiser." } - ] + ], + "expo-image" ] } } diff --git a/mobile/app/index.tsx b/mobile/app/index.tsx index 6639542..f4794a2 100644 --- a/mobile/app/index.tsx +++ b/mobile/app/index.tsx @@ -4,7 +4,7 @@ import { useNavigation } from '@react-navigation/native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { NativeStackNavigationProp } from '@react-navigation/native-stack'; import { MaterialIcons } from '@expo/vector-icons'; -import { useDeleteFile, useAddTags, useMoveFiles, useFolders, useFileImage } from '../hooks/useFiles'; +import { useDeleteFile, useAddTags, useMoveFiles, useFolders } from '../hooks/useFiles'; import { useUnifiedFiles, useFreeLocalSpace } from '../hooks/useUnifiedFiles'; import { UnifiedFileItem } from '../hooks/useUnifiedFiles'; import { FileItem, isFolder } from '../types'; @@ -84,9 +84,7 @@ function formatDateLabel(key: string): string { return key.charAt(0).toUpperCase() + key.slice(1); } -function FileGridItem({ file, onPress, onLongPress, selected }: { file: UnifiedFileItem; onPress?: () => void; onLongPress?: () => void; selected?: boolean }) { - const { data, isLoading } = useFileImage(file.id); - +const FileGridItem = React.memo(function FileGridItem({ file, onPress, onLongPress, selected }: { file: UnifiedFileItem; onPress?: () => void; onLongPress?: () => void; selected?: boolean }) { return ( {selected && ( @@ -114,7 +111,28 @@ function FileGridItem({ file, onPress, onLongPress, selected }: { file: UnifiedF {file.name} ); -} +}); + +const FileGroup = React.memo(function FileGroup({ groupFiles, selectedIds, onItemPress, onItemLongPress }: { + groupFiles: UnifiedFileItem[]; + selectedIds: Set; + onItemPress: (file: UnifiedFileItem) => void; + onItemLongPress: (file: UnifiedFileItem) => void; +}) { + return ( + + {groupFiles.map((file) => ( + onItemPress(file)} + onLongPress={() => onItemLongPress(file)} + /> + ))} + + ); +}); function matchesQuery(file: UnifiedFileItem, query: string, filters: SearchFilters): boolean { if (!query) return true; @@ -447,17 +465,12 @@ export function HomeScreen() { {label} {groupFiles.length} - - {groupFiles.map((file) => ( - handleItemPress(file)} - onLongPress={() => handleItemLongPress(file)} - /> - ))} - + ); }} diff --git a/mobile/components/FileThumbnail.tsx b/mobile/components/FileThumbnail.tsx index 0d7ae1a..527078e 100644 --- a/mobile/components/FileThumbnail.tsx +++ b/mobile/components/FileThumbnail.tsx @@ -1,5 +1,6 @@ import React from 'react'; -import { View, Text, Image, ActivityIndicator, StyleSheet } from 'react-native'; +import { View, Text, ActivityIndicator, StyleSheet } from 'react-native'; +import { Image } from 'expo-image'; import { MaterialIcons } from '@expo/vector-icons'; import type { ComponentProps } from 'react'; import { SyncStatusBadge } from './SyncStatusBadge'; @@ -40,7 +41,7 @@ interface FileThumbnailProps { syncStatus?: SyncStatus; } -export function FileThumbnail({ uri, thumbnailUrl, mimeType, fileName, size, isLoading, syncStatus }: FileThumbnailProps) { +export const FileThumbnail = React.memo(function FileThumbnail({ uri, thumbnailUrl, mimeType, fileName, size, isLoading, syncStatus }: FileThumbnailProps) { const info = getFileInfo(mimeType, fileName); const ext = getExtension(fileName); @@ -57,7 +58,13 @@ export function FileThumbnail({ uri, thumbnailUrl, mimeType, fileName, size, isL if (imageUri) { return ( - + {syncStatus && } ); @@ -72,7 +79,7 @@ export function FileThumbnail({ uri, thumbnailUrl, mimeType, fileName, size, isL {syncStatus && } ); -} +}); const styles = StyleSheet.create({ container: { diff --git a/mobile/components/SyncStatusBadge.tsx b/mobile/components/SyncStatusBadge.tsx index 7f9f36d..4f2d474 100644 --- a/mobile/components/SyncStatusBadge.tsx +++ b/mobile/components/SyncStatusBadge.tsx @@ -10,8 +10,10 @@ interface SyncStatusBadgeProps { const STATUS_CONFIG: Record = { local: { icon: 'phone-android', color: '#757575', bg: 'rgba(245,245,245,0.9)' }, + syncing: { icon: 'sync', color: '#FF9800', bg: 'rgba(255,243,224,0.9)' }, synced: { icon: 'sync', color: '#4CAF50', bg: 'rgba(232,245,233,0.9)' }, cloud: { icon: 'cloud', color: '#1976D2', bg: 'rgba(227,242,253,0.9)' }, + conflict: { icon: 'warning', color: '#E53935', bg: 'rgba(255,235,238,0.9)' }, }; export function SyncStatusBadge({ status, size = 16 }: SyncStatusBadgeProps) { diff --git a/mobile/hooks/useAutoSync.ts b/mobile/hooks/useAutoSync.ts index 5812ee2..0ea00b2 100644 --- a/mobile/hooks/useAutoSync.ts +++ b/mobile/hooks/useAutoSync.ts @@ -91,7 +91,6 @@ export function useAutoSync() { if (pendingFiles.length === 0) return; - console.log(`[useAutoSync] mode=${globalMode}, upload de ${pendingFiles.length} fichier(s)`); setIsSyncing(true); for (const entry of pendingFiles) { @@ -107,15 +106,12 @@ export function useAutoSync() { backendFileId: uploaded.id, syncStatus: 'synced', }); - - console.log(`[useAutoSync] "${entry.name}" uploadé → id=${uploaded.id}`); } catch (err) { - console.error(`[useAutoSync] échec upload "${entry.name}":`, err); + // upload failed, will retry on next cycle } } queryClient.invalidateQueries({ queryKey: ['files'] }); - console.log(`[useAutoSync] sync terminé`); } finally { setIsSyncing(false); isRunning.current = false; @@ -123,9 +119,11 @@ export function useAutoSync() { }, [queryClient]); useEffect(() => { + const timeout = setTimeout(() => { + checkAndSync(); + }, 5_000); const interval = setInterval(checkAndSync, 30_000); - checkAndSync(); - return () => clearInterval(interval); + return () => { clearTimeout(timeout); clearInterval(interval); }; }, [checkAndSync]); return { triggerSync: checkAndSync }; diff --git a/mobile/hooks/useDeviceFiles.ts b/mobile/hooks/useDeviceFiles.ts index 82a98cd..1381d86 100644 --- a/mobile/hooks/useDeviceFiles.ts +++ b/mobile/hooks/useDeviceFiles.ts @@ -74,10 +74,9 @@ async function scanSafFolder(folder: StoredFolder): Promise { }); } return files; - } catch (err) { - console.error(`[useDeviceFiles] SAF scan error for folder "${folder.name}":`, err); - return []; - } + } catch (err) { + return []; + } } export function useDeviceFiles() { @@ -121,7 +120,6 @@ export function useDeviceFiles() { setFiles(deviceFiles); } catch (err) { - console.error('[useDeviceFiles] scan error:', err); setFiles([]); } finally { setIsLoading(false); @@ -132,11 +130,8 @@ export function useDeviceFiles() { const visibleFolders = safDirectory.getVisibleFolders(); if (visibleFolders.length === 0) return; - const safFiles: DeviceFile[] = []; - for (const folder of visibleFolders) { - const folderFiles = await scanSafFolder(folder); - safFiles.push(...folderFiles); - } + const results = await Promise.all(visibleFolders.map((folder) => scanSafFolder(folder))); + const safFiles = results.flat(); setFiles((prev) => { const existing = new Set(prev.filter((f) => !f.folderId).map((f) => f.id)); @@ -191,7 +186,6 @@ export function useDeviceFiles() { }); return true; } catch (err) { - console.error('[useDeviceFiles] pickDirectory error:', err); return false; } }, []); diff --git a/mobile/hooks/useFiles.ts b/mobile/hooks/useFiles.ts index 422148b..a3eb217 100644 --- a/mobile/hooks/useFiles.ts +++ b/mobile/hooks/useFiles.ts @@ -2,14 +2,26 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { apiClient } from '../api/client'; import { ENDPOINTS } from '../constants/api'; import { FileItem, PaginatedResponse } from '../types'; +import { metadataCache } from '../services/metadataCache'; export function useFiles(page: number = 1, limit: number = 20) { return useQuery({ queryKey: ['files', page, limit], - queryFn: () => - apiClient.get>( + queryFn: async () => { + const res = await apiClient.get>( `${ENDPOINTS.FILES}?page=${page}&limit=${limit}&thumbnail=thumbnail` - ), + ); + metadataCache.setFiles(res.data, res.meta.page, res.meta.total); + return res; + }, + initialData: () => { + const cached = metadataCache.getFiles(); + if (cached && cached.page === page) { + return { data: cached.files, meta: { page: cached.page, total: cached.total } }; + } + return undefined; + }, + staleTime: 30_000, }); } @@ -36,6 +48,7 @@ export function useDeleteFile() { mutationFn: (id: string) => apiClient.delete(`${ENDPOINTS.FILES}/${id}`), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['files'] }); + metadataCache.clear(); }, }); } @@ -48,6 +61,7 @@ export function useAddTags() { apiClient.post(`${ENDPOINTS.FILES}/${fileId}/tags`, { tags, tag_type: tagType }), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['files'] }); + metadataCache.clear(); }, }); } @@ -60,6 +74,7 @@ export function useMoveFiles() { apiClient.post(ENDPOINTS.MOVE, { file_ids: fileIds, parent_file_id: parentFileId }), onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['files'] }); + metadataCache.clear(); }, }); } diff --git a/mobile/hooks/useLocalFiles.ts b/mobile/hooks/useLocalFiles.ts index 27b482e..9a1ecb7 100644 --- a/mobile/hooks/useLocalFiles.ts +++ b/mobile/hooks/useLocalFiles.ts @@ -5,13 +5,17 @@ import { LocalFileEntry } from '../types'; export function useLocalFiles() { const { files: deviceFiles, isLoading: deviceLoading, hasPermission, requestPermission, rescan, pickDirectory, folders, refreshFolders } = useDeviceFiles(); - - const registryEntries = useMemo(() => localFileRegistry.getAll(), []); + const lastDeviceCount = useRef(0); useEffect(() => { + if (deviceFiles.length === 0) return; + if (deviceFiles.length === lastDeviceCount.current) return; + lastDeviceCount.current = deviceFiles.length; + + const newEntries: LocalFileEntry[] = []; for (const df of deviceFiles) { if (localFileRegistry.get(df.id)) continue; - const entry: LocalFileEntry = { + newEntries.push({ id: df.id, localUri: df.uri, name: df.name, @@ -20,12 +24,15 @@ export function useLocalFiles() { syncStatus: 'local', createdAt: df.createdAt, folderId: df.folderId, - }; - localFileRegistry.register(entry); + }); + } + if (newEntries.length > 0) { + localFileRegistry.registerBatch(newEntries); } }, [deviceFiles]); const localFiles = useMemo(() => { + const registryEntries = localFileRegistry.getAll(); const merged = new Map(); for (const entry of registryEntries) { @@ -47,10 +54,8 @@ export function useLocalFiles() { } } - const result = Array.from(merged.values()); - console.log(`[useLocalFiles] registry=${registryEntries.length} device=${deviceFiles.length} merged=${result.length}`); - return result; - }, [deviceFiles, registryEntries]); + return Array.from(merged.values()); + }, [deviceFiles]); return { localFiles, diff --git a/mobile/hooks/usePullSync.ts b/mobile/hooks/usePullSync.ts new file mode 100644 index 0000000..dc339c8 --- /dev/null +++ b/mobile/hooks/usePullSync.ts @@ -0,0 +1,83 @@ +import { useCallback, useRef } from 'react'; +import { useQueryClient } from '@tanstack/react-query'; +import { downloadAsync, documentDirectory, makeDirectoryAsync } from 'expo-file-system/legacy'; +import { useFiles } from './useFiles'; +import { localFileRegistry } from '../services/localFileRegistry'; +import { apiClient } from '../api/client'; +import { LocalFileEntry } from '../types'; +import { setIsSyncing } from './useSyncQueue'; + +const SYNC_DIR = `${documentDirectory}synced-files/`; + +export function usePullSync() { + const queryClient = useQueryClient(); + const isRunning = useRef(false); + + const pullNewFiles = useCallback(async () => { + if (isRunning.current) return { pulled: 0 }; + isRunning.current = true; + + try { + setIsSyncing(true); + + const res = await apiClient.get<{ data: Array<{ + id: string; + name: string; + mimeType: string; + size: number; + createdAt: string; + url?: string; + }> }>('/files?page=1&limit=100&thumbnail=thumbnail'); + + const backendFiles = res.data ?? []; + const registry = localFileRegistry.getAll(); + const existingBackendIds = new Set( + registry.filter((e) => e.backendFileId).map((e) => e.backendFileId) + ); + + let pulled = 0; + + for (const bf of backendFiles) { + if (existingBackendIds.has(bf.id)) continue; + if (bf.size === 0) continue; + + try { + const detail = await apiClient.get<{ data: { url: string } }>(`/files/${bf.id}`); + const downloadUrl = detail.data.url; + + await makeDirectoryAsync(SYNC_DIR, { intermediates: true }); + const safeName = bf.name.replace(/[^a-zA-Z0-9._-]/g, '_'); + const fileUri = `${SYNC_DIR}${bf.id}_${safeName}`; + + const result = await downloadAsync(downloadUrl, fileUri); + + const entry: LocalFileEntry = { + id: `pull_${bf.id}`, + backendFileId: bf.id, + localUri: result.uri, + name: bf.name, + mimeType: bf.mimeType, + size: bf.size, + syncStatus: 'synced', + createdAt: bf.createdAt, + }; + localFileRegistry.register(entry); + pulled++; + } catch { + // skip individual file failures + } + } + + if (pulled > 0) { + queryClient.invalidateQueries({ queryKey: ['files'] }); + } + + return { pulled }; + } finally { + setIsSyncing(false); + isRunning.current = false; + } + }, [queryClient]); + + return { pullNewFiles }; +} diff --git a/mobile/hooks/useUnifiedFiles.ts b/mobile/hooks/useUnifiedFiles.ts index a5019de..09cd713 100644 --- a/mobile/hooks/useUnifiedFiles.ts +++ b/mobile/hooks/useUnifiedFiles.ts @@ -4,6 +4,7 @@ import { downloadAsync, documentDirectory, makeDirectoryAsync, deleteAsync } fro import { useFiles } from './useFiles'; import { useLocalFiles } from './useLocalFiles'; import { localFileRegistry } from '../services/localFileRegistry'; +import { thumbnailCache } from '../services/thumbnailCache'; import { apiClient } from '../api/client'; import { FileItem, LocalFileEntry, SyncStatus, Tag } from '../types'; @@ -24,6 +25,7 @@ export interface UnifiedFileItem { url?: string; thumbnailUrl?: string; isDeviceFile?: boolean; + duplicateOf?: string; } const DOWNLOAD_DIR = `${documentDirectory}synced-files/`; @@ -41,6 +43,15 @@ function getExtension(name: string): string { return dot >= 0 ? name.slice(dot) : ''; } +function parseExpiresFromUrl(url: string): number { + try { + const u = new URL(url); + const expires = u.searchParams.get('expires'); + if (expires) return Number(expires) * 1000; + } catch {} + return Date.now() + 50 * 60 * 1000; +} + export function useUnifiedFiles(page: number = 1, limit: number = 50) { const { data: backendData, isLoading: backendLoading, error: backendError } = useFiles(page, limit); const { localFiles, isLoading: localLoading, hasPermission, requestPermission, pickDirectory, folders, refreshFolders } = useLocalFiles(); @@ -57,8 +68,25 @@ export function useUnifiedFiles(page: number = 1, limit: number = 50) { } } + const nameSizeIndex = new Map(); + for (const bf of backendFiles) { + if (!bf.isFolder && bf.size > 0) { + nameSizeIndex.set(`${bf.name}::${bf.size}`, bf.id); + } + } + for (const bf of backendFiles) { const localEntry = backendIdToLocal.get(bf.id); + + let thumbUrl = bf.thumbnailUrl; + if (thumbUrl && bf.thumbnailUrl) { + const expiresAt = parseExpiresFromUrl(bf.thumbnailUrl); + thumbnailCache.set(bf.id, bf.thumbnailUrl, expiresAt); + } else { + const cached = thumbnailCache.get(bf.id); + if (cached) thumbUrl = cached; + } + merged.set(bf.id, { id: bf.id, backendFileId: bf.id, @@ -74,7 +102,7 @@ export function useUnifiedFiles(page: number = 1, limit: number = 50) { isFolder: bf.isFolder, parentFileId: bf.parentFileId, url: bf.url, - thumbnailUrl: bf.thumbnailUrl, + thumbnailUrl: thumbUrl, }); } @@ -82,6 +110,18 @@ export function useUnifiedFiles(page: number = 1, limit: number = 50) { if (lf.backendFileId && merged.has(lf.backendFileId)) continue; if (merged.has(lf.id)) continue; + if (!lf.folderId && lf.size > 0) { + const key = `${lf.name}::${lf.size}`; + const matchId = nameSizeIndex.get(key); + if (matchId) { + const existing = merged.get(matchId); + if (existing && !existing.localUri && lf.localUri) { + merged.set(matchId, { ...existing, localUri: lf.localUri, syncStatus: 'synced' }); + } + continue; + } + } + merged.set(lf.id, { id: lf.id, backendFileId: lf.backendFileId, @@ -138,6 +178,16 @@ export function useUnifiedFilesByParent(parentId: string) { for (const bf of backendFiles) { const localEntry = backendIdToLocal.get(bf.id); + + let thumbUrl = bf.thumbnailUrl; + if (thumbUrl && bf.thumbnailUrl) { + const expiresAt = parseExpiresFromUrl(bf.thumbnailUrl); + thumbnailCache.set(bf.id, bf.thumbnailUrl, expiresAt); + } else { + const cached = thumbnailCache.get(bf.id); + if (cached) thumbUrl = cached; + } + merged.set(bf.id, { id: bf.id, backendFileId: bf.id, @@ -153,7 +203,7 @@ export function useUnifiedFilesByParent(parentId: string) { isFolder: bf.isFolder, parentFileId: bf.parentFileId, url: bf.url, - thumbnailUrl: bf.thumbnailUrl, + thumbnailUrl: thumbUrl, }); } diff --git a/mobile/package-lock.json b/mobile/package-lock.json index 64e48ec..927b7b1 100644 --- a/mobile/package-lock.json +++ b/mobile/package-lock.json @@ -16,6 +16,7 @@ "expo": "~57.0.4", "expo-document-picker": "~57.0.0", "expo-file-system": "~57.0.0", + "expo-image": "~57.0.1", "expo-image-picker": "~57.0.2", "expo-media-library": "~57.0.3", "expo-print": "~57.0.0", @@ -3082,6 +3083,26 @@ "react-native": "*" } }, + "node_modules/expo-image": { + "version": "57.0.1", + "resolved": "https://registry.npmjs.org/expo-image/-/expo-image-57.0.1.tgz", + "integrity": "sha512-EP0lisd2bUqtErry4weRcMW9bLMxtKsht/MLLK3/3do5u4ZMiJbWkY5zfYV+WYmeGab7x9G0sjbeFeEagYGjMw==", + "license": "MIT", + "dependencies": { + "sf-symbols-typescript": "^2.2.0" + }, + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*", + "react-native-web": "*" + }, + "peerDependenciesMeta": { + "react-native-web": { + "optional": true + } + } + }, "node_modules/expo-image-loader": { "version": "57.0.0", "resolved": "https://registry.npmjs.org/expo-image-loader/-/expo-image-loader-57.0.0.tgz", diff --git a/mobile/package.json b/mobile/package.json index 9181dae..c7bf025 100644 --- a/mobile/package.json +++ b/mobile/package.json @@ -11,6 +11,7 @@ "expo": "~57.0.4", "expo-document-picker": "~57.0.0", "expo-file-system": "~57.0.0", + "expo-image": "~57.0.1", "expo-image-picker": "~57.0.2", "expo-media-library": "~57.0.3", "expo-print": "~57.0.0", diff --git a/mobile/services/localFileRegistry.ts b/mobile/services/localFileRegistry.ts index 8083fa6..d6f0e1f 100644 --- a/mobile/services/localFileRegistry.ts +++ b/mobile/services/localFileRegistry.ts @@ -3,58 +3,113 @@ import { LocalFileEntry, SyncStatus } from '../types'; const storage = createMMKV({ id: 'vaultdrop-local-files' }); -const INDEX_KEY = 'local_files_index'; +const BLOB_KEY = 'local_files_v2'; +const LEGACY_INDEX_KEY = 'local_files_index'; -function getAllIds(): string[] { - const raw = storage.getString(INDEX_KEY); - if (!raw) return []; - return JSON.parse(raw) as string[]; +interface RegistryBlob { + entries: Record; + backendIndex: Record; } -function setAllIds(ids: string[]) { - storage.set(INDEX_KEY, JSON.stringify(ids)); +let memoryCache: RegistryBlob | null = null; + +function loadBlob(): RegistryBlob { + if (memoryCache) return memoryCache; + + const raw = storage.getString(BLOB_KEY); + if (raw) { + memoryCache = JSON.parse(raw) as RegistryBlob; + return memoryCache; + } + + memoryCache = migrateFromLegacy(); + saveBlob(memoryCache); + return memoryCache; } -function entryKey(id: string): string { - return `local_file_${id}`; +function saveBlob(blob: RegistryBlob) { + memoryCache = blob; + storage.set(BLOB_KEY, JSON.stringify(blob)); +} + +function migrateFromLegacy(): RegistryBlob { + const blob: RegistryBlob = { entries: {}, backendIndex: {} }; + + const rawIndex = storage.getString(LEGACY_INDEX_KEY); + if (!rawIndex) return blob; + + const ids: string[] = JSON.parse(rawIndex); + for (const id of ids) { + const raw = storage.getString(`local_file_${id}`); + if (!raw) continue; + const entry: LocalFileEntry = JSON.parse(raw); + blob.entries[entry.id] = entry; + if (entry.backendFileId) { + blob.backendIndex[entry.backendFileId] = entry.id; + } + } + + storage.remove(LEGACY_INDEX_KEY); + for (const id of ids) { + storage.remove(`local_file_${id}`); + } + + return blob; } export const localFileRegistry = { register(entry: LocalFileEntry) { - storage.set(entryKey(entry.id), JSON.stringify(entry)); - const ids = getAllIds(); - if (!ids.includes(entry.id)) { - setAllIds([entry.id, ...ids]); + const blob = loadBlob(); + blob.entries[entry.id] = entry; + if (entry.backendFileId) { + blob.backendIndex[entry.backendFileId] = entry.id; } + saveBlob(blob); + }, + + registerBatch(entries: LocalFileEntry[]) { + if (entries.length === 0) return; + const blob = loadBlob(); + for (const entry of entries) { + blob.entries[entry.id] = entry; + if (entry.backendFileId) { + blob.backendIndex[entry.backendFileId] = entry.id; + } + } + saveBlob(blob); }, get(id: string): LocalFileEntry | undefined { - const raw = storage.getString(entryKey(id)); - if (!raw) return undefined; - return JSON.parse(raw) as LocalFileEntry; + return loadBlob().entries[id]; }, getByBackendId(backendId: string): LocalFileEntry | undefined { - const ids = getAllIds(); - for (const id of ids) { - const entry = this.get(id); - if (entry?.backendFileId === backendId) return entry; - } - return undefined; + const blob = loadBlob(); + const entryId = blob.backendIndex[backendId]; + if (!entryId) return undefined; + return blob.entries[entryId]; }, getAll(): LocalFileEntry[] { - const ids = getAllIds(); - return ids - .map((id) => this.get(id)) - .filter((e): e is LocalFileEntry => e !== undefined); + const blob = loadBlob(); + return Object.values(blob.entries); }, update(id: string, updates: Partial) { - const existing = this.get(id); + const blob = loadBlob(); + const existing = blob.entries[id]; if (!existing) return; + + if (existing.backendFileId && updates.backendFileId === undefined && updates.syncStatus === 'cloud') { + delete blob.backendIndex[existing.backendFileId]; + } + const updated = { ...existing, ...updates }; - storage.set(entryKey(id), JSON.stringify(updated)); + blob.entries[id] = updated; + if (updated.backendFileId) { + blob.backendIndex[updated.backendFileId] = id; + } + saveBlob(blob); }, updateSyncStatus(id: string, syncStatus: SyncStatus) { @@ -66,17 +121,26 @@ export const localFileRegistry = { }, remove(id: string) { - storage.remove(entryKey(id)); - const ids = getAllIds().filter((i) => i !== id); - setAllIds(ids); + const blob = loadBlob(); + const entry = blob.entries[id]; + if (entry?.backendFileId) { + delete blob.backendIndex[entry.backendFileId]; + } + delete blob.entries[id]; + saveBlob(blob); }, removeByBackendId(backendId: string) { - const entry = this.getByBackendId(backendId); - if (entry) this.remove(entry.id); + const blob = loadBlob(); + const entryId = blob.backendIndex[backendId]; + if (entryId) { + delete blob.entries[entryId]; + delete blob.backendIndex[backendId]; + saveBlob(blob); + } }, count(): number { - return getAllIds().length; + return Object.keys(loadBlob().entries).length; }, }; diff --git a/mobile/services/metadataCache.ts b/mobile/services/metadataCache.ts new file mode 100644 index 0000000..c49a7e6 --- /dev/null +++ b/mobile/services/metadataCache.ts @@ -0,0 +1,44 @@ +import { createMMKV } from 'react-native-mmkv'; +import { FileItem } from '../types'; + +const storage = createMMKV({ id: 'vaultdrop-metadata' }); + +const FILES_KEY = 'backend_files_cache'; +const UPDATED_AT_KEY = 'cache_updated_at'; +const STALE_MS = 5 * 60 * 1000; + +interface CachedFiles { + files: FileItem[]; + page: number; + total: number; +} + +export const metadataCache = { + getFiles(): CachedFiles | null { + const raw = storage.getString(FILES_KEY); + if (!raw) return null; + try { + return JSON.parse(raw) as CachedFiles; + } catch { + return null; + } + }, + + setFiles(files: FileItem[], page: number, total: number) { + const data: CachedFiles = { files, page, total }; + storage.set(FILES_KEY, JSON.stringify(data)); + storage.set(UPDATED_AT_KEY, Date.now()); + }, + + isStale(): boolean { + const raw = storage.getString(UPDATED_AT_KEY); + if (!raw) return true; + const updatedAt = Number(raw); + return Date.now() - updatedAt > STALE_MS; + }, + + clear() { + storage.remove(FILES_KEY); + storage.remove(UPDATED_AT_KEY); + }, +}; diff --git a/mobile/services/thumbnailCache.ts b/mobile/services/thumbnailCache.ts new file mode 100644 index 0000000..37347c4 --- /dev/null +++ b/mobile/services/thumbnailCache.ts @@ -0,0 +1,65 @@ +import { createMMKV } from 'react-native-mmkv'; + +const storage = createMMKV({ id: 'vaultdrop-thumbnails' }); + +const BLOB_KEY = 'thumbnail_urls'; +const STALE_MS = 50 * 60 * 1000; + +interface ThumbnailCacheEntry { + url: string; + expiresAt: number; +} + +let memoryCache: Record | null = null; + +function loadCache(): Record { + if (memoryCache) return memoryCache; + const raw = storage.getString(BLOB_KEY); + memoryCache = raw ? JSON.parse(raw) : {}; + return memoryCache!; +} + +function saveCache(cache: Record) { + memoryCache = cache; + storage.set(BLOB_KEY, JSON.stringify(cache)); +} + +export const thumbnailCache = { + get(fileId: string): string | null { + const cache = loadCache(); + const entry = cache[fileId]; + if (!entry) return null; + if (Date.now() > entry.expiresAt) { + delete cache[fileId]; + saveCache(cache); + return null; + } + return entry.url; + }, + + set(fileId: string, url: string, expiresAt: number) { + const cache = loadCache(); + cache[fileId] = { url, expiresAt }; + saveCache(cache); + }, + + setBatch(entries: Array<{ fileId: string; url: string; expiresAt: number }>) { + if (entries.length === 0) return; + const cache = loadCache(); + for (const e of entries) { + cache[e.fileId] = { url: e.url, expiresAt: e.expiresAt }; + } + saveCache(cache); + }, + + remove(fileId: string) { + const cache = loadCache(); + delete cache[fileId]; + saveCache(cache); + }, + + clear() { + memoryCache = {}; + storage.remove(BLOB_KEY); + }, +}; diff --git a/mobile/types/index.ts b/mobile/types/index.ts index 7a15f0a..2f596db 100644 --- a/mobile/types/index.ts +++ b/mobile/types/index.ts @@ -117,7 +117,7 @@ export interface RefreshResponse { refresh_token: string; } -export type SyncStatus = 'local' | 'synced' | 'cloud'; +export type SyncStatus = 'local' | 'syncing' | 'synced' | 'cloud' | 'conflict'; export interface LocalFileEntry { id: string;