diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 8826d55..96d05da 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -51,7 +51,7 @@ func main() { conversionSvc.Start(cfg.ConversionWorkers) defer conversionSvc.Stop() - h := handler.New(resourceSvc, ocrSvc, urlSvc, auth.NewAuthHandler(authSvc), conversionSvc, rebacSvc, placementSvc, syncSvc, eventBroker) + h := handler.New(database, resourceSvc, ocrSvc, urlSvc, auth.NewAuthHandler(authSvc), conversionSvc, rebacSvc, placementSvc, syncSvc, eventBroker) r := gin.Default() handler.SetupRoutes(r, h, authSvc) diff --git a/backend/internal/db/queries/resources.sql b/backend/internal/db/queries/resources.sql index 66e4971..bd1b63c 100644 --- a/backend/internal/db/queries/resources.sql +++ b/backend/internal/db/queries/resources.sql @@ -56,12 +56,22 @@ SELECT * FROM resources WHERE checksum = $1 AND is_folder = false AND owner_id = $2 LIMIT 1; +-- name: CountResourcesByOwner :one +SELECT COUNT(*) FROM resources +WHERE owner_id = $1 AND parent_resource_id IS NULL; + -- name: ListResourcesByOwner :many SELECT * FROM resources WHERE owner_id = $1 AND parent_resource_id IS NULL -ORDER BY created_at DESC; +ORDER BY created_at DESC +LIMIT $2 OFFSET $3; + +-- name: CountResourcesByParentAndOwner :one +SELECT COUNT(*) FROM resources +WHERE parent_resource_id = $1 AND owner_id = $2; -- name: ListResourcesByParentAndOwner :many SELECT * FROM resources WHERE parent_resource_id = $1 AND owner_id = $2 -ORDER BY is_folder DESC, created_at DESC; +ORDER BY is_folder DESC, created_at DESC +LIMIT $3 OFFSET $4; diff --git a/backend/internal/db/resources.sql.go b/backend/internal/db/resources.sql.go index 9428403..94d5351 100644 --- a/backend/internal/db/resources.sql.go +++ b/backend/internal/db/resources.sql.go @@ -13,6 +13,35 @@ import ( "github.com/lib/pq" ) +const countResourcesByOwner = `-- name: CountResourcesByOwner :one +SELECT COUNT(*) FROM resources +WHERE owner_id = $1 AND parent_resource_id IS NULL +` + +func (q *Queries) CountResourcesByOwner(ctx context.Context, ownerID uuid.UUID) (int64, error) { + row := q.db.QueryRowContext(ctx, countResourcesByOwner, ownerID) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countResourcesByParentAndOwner = `-- name: CountResourcesByParentAndOwner :one +SELECT COUNT(*) FROM resources +WHERE parent_resource_id = $1 AND owner_id = $2 +` + +type CountResourcesByParentAndOwnerParams struct { + ParentResourceID uuid.NullUUID `json:"parent_resource_id"` + OwnerID uuid.UUID `json:"owner_id"` +} + +func (q *Queries) CountResourcesByParentAndOwner(ctx context.Context, arg CountResourcesByParentAndOwnerParams) (int64, error) { + row := q.db.QueryRowContext(ctx, countResourcesByParentAndOwner, arg.ParentResourceID, arg.OwnerID) + var count int64 + err := row.Scan(&count) + return count, err +} + const createFolder = `-- name: CreateFolder :one INSERT INTO resources (name, is_folder, owner_id, created_at, updated_at) VALUES ($1, true, $2, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) @@ -323,10 +352,17 @@ const listResourcesByOwner = `-- name: ListResourcesByOwner :many SELECT id, name, mime_type, size, checksum, ocr_text, is_folder, parent_resource_id, owner_id, created_at, updated_at FROM resources WHERE owner_id = $1 AND parent_resource_id IS NULL ORDER BY created_at DESC +LIMIT $2 OFFSET $3 ` -func (q *Queries) ListResourcesByOwner(ctx context.Context, ownerID uuid.UUID) ([]Resource, error) { - rows, err := q.db.QueryContext(ctx, listResourcesByOwner, ownerID) +type ListResourcesByOwnerParams struct { + OwnerID uuid.UUID `json:"owner_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` +} + +func (q *Queries) ListResourcesByOwner(ctx context.Context, arg ListResourcesByOwnerParams) ([]Resource, error) { + rows, err := q.db.QueryContext(ctx, listResourcesByOwner, arg.OwnerID, arg.Limit, arg.Offset) if err != nil { return nil, err } @@ -364,15 +400,23 @@ const listResourcesByParentAndOwner = `-- name: ListResourcesByParentAndOwner :m SELECT id, name, mime_type, size, checksum, ocr_text, is_folder, parent_resource_id, owner_id, created_at, updated_at FROM resources WHERE parent_resource_id = $1 AND owner_id = $2 ORDER BY is_folder DESC, created_at DESC +LIMIT $3 OFFSET $4 ` type ListResourcesByParentAndOwnerParams struct { ParentResourceID uuid.NullUUID `json:"parent_resource_id"` OwnerID uuid.UUID `json:"owner_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` } func (q *Queries) ListResourcesByParentAndOwner(ctx context.Context, arg ListResourcesByParentAndOwnerParams) ([]Resource, error) { - rows, err := q.db.QueryContext(ctx, listResourcesByParentAndOwner, arg.ParentResourceID, arg.OwnerID) + rows, err := q.db.QueryContext(ctx, listResourcesByParentAndOwner, + arg.ParentResourceID, + arg.OwnerID, + arg.Limit, + arg.Offset, + ) if err != nil { return nil, err } diff --git a/backend/internal/handler/handler.go b/backend/internal/handler/handler.go index 3c18699..bf22be4 100644 --- a/backend/internal/handler/handler.go +++ b/backend/internal/handler/handler.go @@ -1,6 +1,8 @@ package handler import ( + "database/sql" + "github.com/vaultdrop/backend/internal/auth" "github.com/vaultdrop/backend/internal/service" ) @@ -17,6 +19,7 @@ type Handler struct { } func New( + database *sql.DB, resourceSvc *service.ResourceService, ocrSvc *service.OCRService, urlSvc *service.URLService, @@ -36,7 +39,7 @@ func New( broker: eventBroker, }, OCR: &OCRHandler{ocr: ocrSvc, resources: resourceSvc}, - Health: &HealthHandler{ocr: ocrSvc}, + Health: &HealthHandler{db: database, ocr: ocrSvc}, Auth: authHandler, Share: &ShareHandler{rebac: rebacSvc}, Device: &DeviceHandler{placement: placementSvc}, diff --git a/backend/internal/handler/health.go b/backend/internal/handler/health.go index 44f5463..be09a68 100644 --- a/backend/internal/handler/health.go +++ b/backend/internal/handler/health.go @@ -1,14 +1,44 @@ package handler import ( + "database/sql" + "github.com/gin-gonic/gin" "github.com/vaultdrop/backend/pkg/api" ) type HealthHandler struct { + db *sql.DB ocr interface{ HealthCheck() error } } func (h *HealthHandler) Check(c *gin.Context) { - api.Success(c, gin.H{"status": "healthy"}) + checks := gin.H{} + + dbErr := h.db.Ping() + if dbErr != nil { + checks["database"] = "error: " + dbErr.Error() + } else { + checks["database"] = "ok" + } + + ocrErr := h.ocr.HealthCheck() + if ocrErr != nil { + checks["ocr"] = "error: " + ocrErr.Error() + } else { + checks["ocr"] = "ok" + } + + status := "healthy" + for _, v := range checks { + if v != "ok" { + status = "degraded" + break + } + } + + api.Success(c, gin.H{ + "status": status, + "checks": checks, + }) } diff --git a/backend/internal/handler/resource.go b/backend/internal/handler/resource.go index 25b9aa0..402a17a 100644 --- a/backend/internal/handler/resource.go +++ b/backend/internal/handler/resource.go @@ -65,11 +65,31 @@ func (h *ResourceHandler) Upload(c *gin.Context) { api.Success(c, results) } +func parsePagination(c *gin.Context) (page, limit int) { + page = 1 + limit = 20 + if p := c.Query("page"); p != "" { + if n, err := strconv.Atoi(p); err == nil && n > 0 { + page = n + } + } + if l := c.Query("limit"); l != "" { + if n, err := strconv.Atoi(l); err == nil && n > 0 { + if n > 100 { + n = 100 + } + limit = n + } + } + return +} + func (h *ResourceHandler) List(c *gin.Context) { userID := c.GetString(auth.UserIDKey) thumbnailQuality := c.Query("thumbnail") + page, limit := parsePagination(c) - resources, err := h.resources.List(userID) + resources, total, err := h.resources.List(userID, page, limit) if err != nil { log.Printf("ERROR List resources: %v", err) api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to list resources") @@ -132,7 +152,7 @@ func (h *ResourceHandler) List(c *gin.Context) { } } - api.Paginated(c, resp, 1, len(resp)) + api.Paginated(c, resp, page, total) } func (h *ResourceHandler) Download(c *gin.Context) { @@ -335,8 +355,9 @@ func (h *ResourceHandler) ListByParent(c *gin.Context) { userID := c.GetString(auth.UserIDKey) parentID := c.Param("id") thumbnailQuality := c.Query("thumbnail") + page, limit := parsePagination(c) - resources, err := h.resources.ListResourcesByParentID(parentID, userID) + resources, total, err := h.resources.ListResourcesByParentID(parentID, userID, page, limit) if err != nil { api.Error(c, http.StatusInternalServerError, "DB_ERROR", "Failed to list resources in folder") return @@ -396,7 +417,7 @@ func (h *ResourceHandler) ListByParent(c *gin.Context) { } } - api.Success(c, resp) + api.Paginated(c, resp, page, total) } func (h *ResourceHandler) GetVariants(c *gin.Context) { diff --git a/backend/internal/service/resource.go b/backend/internal/service/resource.go index 6925177..4a41d66 100644 --- a/backend/internal/service/resource.go +++ b/backend/internal/service/resource.go @@ -147,22 +147,33 @@ func (s *ResourceService) ensureServerPlacementQtx(q *db.Queries, resourceID, ow return placement, nil } -func (s *ResourceService) List(ownerID string) ([]model.Resource, error) { +func (s *ResourceService) List(ownerID string, page, limit int) ([]model.Resource, int, error) { ownerUUID, _ := uuid.Parse(ownerID) - dbResources, err := s.queries.ListResourcesByOwner(context.Background(), ownerUUID) + + total, err := s.queries.CountResourcesByOwner(context.Background(), ownerUUID) if err != nil { - return nil, fmt.Errorf("list resources: %w", err) + return nil, 0, fmt.Errorf("count resources: %w", err) + } + + offset := (page - 1) * limit + dbResources, err := s.queries.ListResourcesByOwner(context.Background(), db.ListResourcesByOwnerParams{ + OwnerID: ownerUUID, + Limit: int32(limit), + Offset: int32(offset), + }) + if err != nil { + return nil, 0, fmt.Errorf("list resources: %w", err) } resources := make([]model.Resource, len(dbResources)) for i, r := range dbResources { tags, err := s.queries.GetTagsByResourceID(context.Background(), r.ID) if err != nil { - return nil, fmt.Errorf("get tags for resource %s: %w", r.ID, err) + return nil, 0, fmt.Errorf("get tags for resource %s: %w", r.ID, err) } resources[i] = dbResourceToModel(r, tags) } - return resources, nil + return resources, int(total), nil } func (s *ResourceService) Get(id string) (*model.Resource, error) { @@ -309,25 +320,37 @@ func (s *ResourceService) ListFolders(ownerID string) ([]model.Resource, error) return folders, nil } -func (s *ResourceService) ListResourcesByParentID(parentID, ownerID string) ([]model.Resource, error) { +func (s *ResourceService) ListResourcesByParentID(parentID, ownerID string, page, limit int) ([]model.Resource, int, error) { parentUUID, _ := uuid.Parse(parentID) ownerUUID, _ := uuid.Parse(ownerID) - dbResources, err := s.queries.ListResourcesByParentAndOwner(context.Background(), db.ListResourcesByParentAndOwnerParams{ + + total, err := s.queries.CountResourcesByParentAndOwner(context.Background(), db.CountResourcesByParentAndOwnerParams{ ParentResourceID: uuid.NullUUID{UUID: parentUUID, Valid: true}, OwnerID: ownerUUID, }) if err != nil { - return nil, fmt.Errorf("list resources by parent: %w", err) + return nil, 0, fmt.Errorf("count resources by parent: %w", err) + } + + offset := (page - 1) * limit + dbResources, err := s.queries.ListResourcesByParentAndOwner(context.Background(), db.ListResourcesByParentAndOwnerParams{ + ParentResourceID: uuid.NullUUID{UUID: parentUUID, Valid: true}, + OwnerID: ownerUUID, + Limit: int32(limit), + Offset: int32(offset), + }) + if err != nil { + return nil, 0, fmt.Errorf("list resources by parent: %w", err) } resources := make([]model.Resource, len(dbResources)) for i, r := range dbResources { tags, err := s.queries.GetTagsByResourceID(context.Background(), r.ID) if err != nil { - return nil, fmt.Errorf("get tags for resource %s: %w", r.ID, err) + return nil, 0, fmt.Errorf("get tags for resource %s: %w", r.ID, err) } resources[i] = dbResourceToModel(r, tags) } - return resources, nil + return resources, int(total), nil } func (s *ResourceService) GetVariantsByResourceID(resourceID string) ([]model.Variant, error) { diff --git a/mobile/app/folder.tsx b/mobile/app/folder.tsx index bd7dd01..0a5a52c 100644 --- a/mobile/app/folder.tsx +++ b/mobile/app/folder.tsx @@ -22,6 +22,7 @@ import { deleteAsync } from 'expo-file-system/legacy'; const NUM_COLUMNS = 3; const SCREEN_WIDTH = Dimensions.get('window').width; const ITEM_SIZE = (SCREEN_WIDTH - 16 * 2 - (NUM_COLUMNS - 1) * 6) / NUM_COLUMNS; +const PAGE_SIZE = 100; type RootStackParamList = { Folder: { folderId: string; folderName: string }; @@ -75,7 +76,8 @@ export function FolderScreen() { const route = useRoute(); const navigation = useNavigation(); const { folderId, folderName } = route.params; - const { data, isLoading } = useFiles(folderId); + const [page, setPage] = useState(1); + const { data, isLoading, isFetching } = useFiles(folderId, page, PAGE_SIZE); const deleteFile = useDeleteFile(); const freeLocalSpace = useFreeLocalSpace(); const queryClient = useQueryClient(); @@ -87,6 +89,17 @@ export function FolderScreen() { const { data: foldersData } = useFolders(); const insets = useSafeAreaInsets(); + const loadMore = useCallback(() => { + if (isFetching) return; + const total = data?.meta?.total ?? 0; + const loaded = data?.data?.length ?? 0; + if (loaded < total) { + setPage((p) => p + 1); + } + }, [isFetching, data?.meta?.total, data?.data?.length]); + + const hasMore = (data?.data?.length ?? 0) > 0 && (data?.data?.length ?? 0) < (data?.meta?.total ?? 0); + const [tagModalVisible, setTagModalVisible] = useState(false); const [tagModalMode, setTagModalMode] = useState<'tag' | 'folder'>('tag'); const [tagInput, setTagInput] = useState(''); @@ -246,12 +259,25 @@ export function FolderScreen() { data={files} keyExtractor={(item) => item.id} contentContainerStyle={styles.list} + onEndReached={loadMore} + onEndReachedThreshold={0.5} ListEmptyComponent={ Dossier vide } + ListFooterComponent={ + isFetching ? ( + + Chargement... + + ) : hasMore ? ( + + Charger plus + + ) : null + } renderItem={({ item: file }) => ( { if (parentId) { - const backendRes = await apiClient.get<{ data: FileItem[] }>( - `${ENDPOINTS.RESOURCES}/folders/${parentId}/resources?thumbnail=thumbnail_small`, + const backendRes = await apiClient.get>( + `${ENDPOINTS.RESOURCES}/folders/${parentId}/resources?page=${page}&limit=${limit}&thumbnail=thumbnail_small`, ); fileStore.mergeFromBackend( backendRes.data.map((f) => ({ @@ -69,7 +69,7 @@ export function useFiles(parentId?: string | null, page: number = 1, limit: numb const children = fileStore.getChildrenByParent(parentId); return { data: children.map((r) => recordToUnifiedItem(r)!).filter(Boolean), - meta: { page: 0, total: children.length }, + meta: { page, total: backendRes.meta?.total ?? children.length }, }; } diff --git a/mobile/hooks/usePullSync.ts b/mobile/hooks/usePullSync.ts index ba61a93..2c45031 100644 --- a/mobile/hooks/usePullSync.ts +++ b/mobile/hooks/usePullSync.ts @@ -19,7 +19,10 @@ export function usePullSync() { try { setIsSyncing(true); - const res = await apiClient.get<{ data: Array<{ + let page = 1; + const limit = 100; + let total = 0; + const backendResources: Array<{ id: string; name: string; mimeType: string; @@ -28,9 +31,17 @@ export function usePullSync() { url?: string; thumbnailUrl?: string; ownerId?: string; - }> }>(`${ENDPOINTS.RESOURCES}?page=1&limit=100&thumbnail=thumbnail_small`); + }> = []; + + do { + const res = await apiClient.get<{ data: Array; meta?: { total: number } }>( + `${ENDPOINTS.RESOURCES}?page=${page}&limit=${limit}&thumbnail=thumbnail_small`, + ); + backendResources.push(...(res.data ?? [])); + total = res.meta?.total ?? res.data.length; + page++; + } while (backendResources.length < total); - const backendResources = res.data ?? []; const registry = fileStore.getAllSynced(); const existingBackendIds = new Set( registry.filter((e) => e.backendId).map((e) => e.backendId) diff --git a/mobile/services/uploadQueue.ts b/mobile/services/uploadQueue.ts index 7b7bde6..62cfc15 100644 --- a/mobile/services/uploadQueue.ts +++ b/mobile/services/uploadQueue.ts @@ -7,6 +7,9 @@ import { ApiError, UploadError } from '../types'; export type UploadFile = { uri: string; type: string; name: string }; export type UploadResult = { name: string; id: string }; +export const UPLOAD_MAX_RETRIES = 3; +const BASE_RETRY_DELAY_MS = 1000; + export type UploadTaskStatus = 'pending' | 'uploading' | 'done' | 'error'; export type UploadTask = { @@ -16,6 +19,7 @@ export type UploadTask = { progress: number; result?: UploadResult; error?: string; + retryCount: number; createdAt: number; updatedAt: number; }; @@ -33,6 +37,7 @@ function serialize(task: UploadTask): unknown { progress: task.progress, result: task.result ?? null, error: task.error ?? null, + retryCount: task.retryCount, createdAt: task.createdAt, updatedAt: task.updatedAt, }; @@ -50,6 +55,7 @@ function deserialize(data: unknown): UploadTask | null { progress: d.progress as number, result: d.result ? (d.result as UploadResult) : undefined, error: d.error ? (d.error as string) : undefined, + retryCount: (d.retryCount as number) ?? 0, createdAt: d.createdAt as number, updatedAt: d.updatedAt as number, }; @@ -147,6 +153,7 @@ class UploadQueue { file, status: 'pending', progress: 0, + retryCount: 0, createdAt: now, updatedAt: now, }); @@ -171,6 +178,7 @@ class UploadQueue { if (!task || task.status !== 'error') return; task.status = 'pending'; task.progress = 0; + task.retryCount = 0; task.error = undefined; task.result = undefined; task.updatedAt = Date.now(); @@ -184,6 +192,7 @@ class UploadQueue { if (task.status === 'error') { task.status = 'pending'; task.progress = 0; + task.retryCount = 0; task.error = undefined; task.result = undefined; task.updatedAt = Date.now(); @@ -212,6 +221,7 @@ class UploadQueue { } private async runTask(task: UploadTask) { + let willRetry = false; try { const fsFile = new File(task.file.uri); const headers: Record = {}; @@ -251,20 +261,36 @@ class UploadQueue { this.notify(); this.scheduleCleanup(); } catch (err) { - task.status = 'error'; - task.error = - err instanceof UploadError - ? `${err.fileName} : ${err.message}` - : err instanceof Error - ? err.message - : 'Erreur inconnue'; - task.updatedAt = Date.now(); - this.persist(); - this.notify(); + task.retryCount++; + if (task.retryCount <= UPLOAD_MAX_RETRIES) { + willRetry = true; + task.status = 'pending'; + task.progress = 0; + task.error = undefined; + task.updatedAt = Date.now(); + this.persist(); + this.notify(); + const delay = BASE_RETRY_DELAY_MS * Math.pow(2, task.retryCount - 1); + setTimeout(() => this.processNext(), delay); + } else { + task.status = 'error'; + task.error = + err instanceof UploadError + ? `${err.fileName} : ${err.message}` + : err instanceof Error + ? err.message + : 'Erreur inconnue'; + task.error += ` (${task.retryCount} tentative(s))`; + task.updatedAt = Date.now(); + this.persist(); + this.notify(); + } } finally { this.active--; this.notify(); - this.processNext(); + if (!willRetry) { + this.processNext(); + } } }