remove npl

backend/cmd/server/main.go

@@ -27,19 +27,13 @@ func main() {
 	queries := db.New(database)
 
 	fileSvc := service.NewFileService(queries, cfg)
-	nlpSvc := service.NewNLPService(cfg, fileSvc)
 	ocrSvc := service.NewOCRService(cfg, fileSvc)
 	urlSvc := service.NewURLService(cfg.HMACSecret, cfg.ServerHost)
 
-	ocrSvc.SetNLPService(nlpSvc)
-
 	ocrSvc.Start()
 	defer ocrSvc.Stop()
 
-	nlpSvc.Start()
+	h := handler.New(fileSvc, ocrSvc, urlSvc)
-	defer nlpSvc.Stop()
-
-	h := handler.New(fileSvc, ocrSvc, nlpSvc, urlSvc)
 
 	r := gin.Default()
 	handler.SetupRoutes(r, h)

backend/internal/config/config.go

@@ -6,7 +6,6 @@ type Config struct {
 	Port        string
 	DBPath      string
 	OCREndpoint string
-	NLPEndpoint string
 	UploadDir   string
 	HMACSecret  string
 	ServerHost  string
@@ -17,7 +16,6 @@ func Load() *Config {
 		Port:        envOr("PORT", "8080"),
 		DBPath:      envOr("DB_PATH", "vaultdrop.db"),
 		OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"),
-		NLPEndpoint: envOr("NLP_ENDPOINT", "http://localhost:9091"),
 		UploadDir:   envOr("UPLOAD_DIR", "./uploads"),
 		HMACSecret:  envOr("HMAC_SECRET", "thisismyrandomstring"),
 		ServerHost:  envOr("SERVER_HOST", "http://192.168.1.17:8080"),

backend/internal/db/files.sql.go

@@ -12,7 +12,7 @@ import (
 const createFile = `-- name: CreateFile :one
 INSERT INTO files (id, name, mime_type, size, storage_key, checksum, created_at, updated_at)
 VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-RETURNING id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data
+RETURNING id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at
 `
 
 type CreateFileParams struct {
@@ -44,7 +44,6 @@ func (q *Queries) CreateFile(ctx context.Context, arg CreateFileParams) (File, e
 		&i.OcrText,
 		&i.CreatedAt,
 		&i.UpdatedAt,
-		&i.NlpData,
 	)
 	return i, err
 }
@@ -60,7 +59,7 @@ func (q *Queries) DeleteFile(ctx context.Context, id string) error {
 }
 
 const getFile = `-- name: GetFile :one
-SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
+SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
 WHERE id = ? LIMIT 1
 `
 
@@ -77,13 +76,12 @@ func (q *Queries) GetFile(ctx context.Context, id string) (File, error) {
 		&i.OcrText,
 		&i.CreatedAt,
 		&i.UpdatedAt,
-		&i.NlpData,
 	)
 	return i, err
 }
 
 const listFiles = `-- name: ListFiles :many
-SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
+SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
 ORDER BY created_at DESC
 `
 
@@ -106,7 +104,6 @@ func (q *Queries) ListFiles(ctx context.Context) ([]File, error) {
 			&i.OcrText,
 			&i.CreatedAt,
 			&i.UpdatedAt,
-			&i.NlpData,
 		); err != nil {
 			return nil, err
 		}
@@ -122,7 +119,7 @@ func (q *Queries) ListFiles(ctx context.Context) ([]File, error) {
 }
 
 const listFilesByID = `-- name: ListFilesByID :many
-SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
+SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
 WHERE id IN (SELECT value FROM json_each(?))
 ORDER BY created_at DESC
 `
@@ -146,7 +143,6 @@ func (q *Queries) ListFilesByID(ctx context.Context, jsonEach interface{}) ([]Fi
 			&i.OcrText,
 			&i.CreatedAt,
 			&i.UpdatedAt,
-			&i.NlpData,
 		); err != nil {
 			return nil, err
 		}
@@ -183,19 +179,3 @@ func (q *Queries) UpdateFile(ctx context.Context, arg UpdateFileParams) error {
 	)
 	return err
 }
-
-const updateNLPData = `-- name: UpdateNLPData :exec
-UPDATE files
-SET nlp_data = ?, updated_at = CURRENT_TIMESTAMP
-WHERE id = ?
-`
-
-type UpdateNLPDataParams struct {
-	NlpData string `json:"nlp_data"`
-	ID      string `json:"id"`
-}
-
-func (q *Queries) UpdateNLPData(ctx context.Context, arg UpdateNLPDataParams) error {
-	_, err := q.db.ExecContext(ctx, updateNLPData, arg.NlpData, arg.ID)
-	return err
-}

backend/internal/db/migrations/003_nlp_data.down.sql

@@ -1 +0,0 @@
-ALTER TABLE files DROP COLUMN nlp_data;

backend/internal/db/migrations/003_nlp_data.up.sql

@@ -1 +0,0 @@
-ALTER TABLE files ADD COLUMN nlp_data TEXT NOT NULL DEFAULT '';

backend/internal/db/models.go

@@ -18,5 +18,4 @@ type File struct {
 	OcrText    string    `json:"ocr_text"`
 	CreatedAt  time.Time `json:"created_at"`
 	UpdatedAt  time.Time `json:"updated_at"`
-	NlpData    string    `json:"nlp_data"`
 }

backend/internal/db/queries/files.sql

@@ -24,8 +24,3 @@ WHERE id = ?;
 -- name: DeleteFile :exec
 DELETE FROM files
 WHERE id = ?;
-
--- name: UpdateNLPData :exec
-UPDATE files
-SET nlp_data = ?, updated_at = CURRENT_TIMESTAMP
-WHERE id = ?;

backend/internal/handler/files.go

@@ -64,7 +64,6 @@ func (h *FileHandler) List(c *gin.Context) {
 		CreatedAt string   `json:"createdAt"`
 		MimeType  string   `json:"mimeType"`
 		OcrText   string   `json:"ocrText,omitempty"`
-		NlpData   string   `json:"nlpData,omitempty"`
 		UpdatedAt string   `json:"updatedAt"`
 	}
 
@@ -79,7 +78,6 @@ func (h *FileHandler) List(c *gin.Context) {
 			Tags:      []string{},
 			CreatedAt: f.CreatedAt,
 			OcrText:   f.OcrText,
-			NlpData:   f.NlpData,
 			UpdatedAt: f.UpdatedAt,
 			MimeType:  f.MimeType,
 		}
@@ -124,7 +122,6 @@ func (h *FileHandler) Get(c *gin.Context) {
 		"createdAt": file.CreatedAt,
 		"updatedAt": file.UpdatedAt,
 		"ocrText":   file.OcrText,
-		"nlpData":   file.NlpData,
 	})
 }
 

backend/internal/handler/handler.go

@@ -7,15 +7,13 @@ import (
 type Handler struct {
 	File   *FileHandler
 	OCR    *OCRHandler
-	NLP    *NLPHandler
 	Health *HealthHandler
 }
 
-func New(fileSvc *service.FileService, ocrSvc *service.OCRService, nlpSvc *service.NLPService, urlSvc *service.URLService) *Handler {
+func New(fileSvc *service.FileService, ocrSvc *service.OCRService, urlSvc *service.URLService) *Handler {
 	return &Handler{
 		File:   &FileHandler{files: fileSvc, urls: urlSvc, ocr: ocrSvc},
 		OCR:    &OCRHandler{ocr: ocrSvc, files: fileSvc},
-		NLP:    &NLPHandler{nlp: nlpSvc, files: fileSvc},
+		Health: &HealthHandler{ocr: ocrSvc},
-		Health: &HealthHandler{ocr: ocrSvc, nlp: nlpSvc},
 	}
 }

backend/internal/handler/health.go

@@ -7,23 +7,8 @@ import (
 
 type HealthHandler struct {
 	ocr interface{ HealthCheck() error }
-	nlp interface{ HealthCheck() error }
 }
 
 func (h *HealthHandler) Check(c *gin.Context) {
-	status := gin.H{"status": "healthy"}
+	api.Success(c, gin.H{"status": "healthy"})
-
-	if err := h.ocr.HealthCheck(); err != nil {
-		status["ocr"] = err.Error()
-	} else {
-		status["ocr"] = "healthy"
-	}
-
-	if err := h.nlp.HealthCheck(); err != nil {
-		status["nlp"] = err.Error()
-	} else {
-		status["nlp"] = "healthy"
-	}
-
-	api.Success(c, status)
 }

backend/internal/handler/nlp.go

@@ -1,22 +0,0 @@
-package handler
-
-import (
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-	"github.com/vaultdrop/backend/internal/service"
-	"github.com/vaultdrop/backend/pkg/api"
-)
-
-type NLPHandler struct {
-	nlp   *service.NLPService
-	files *service.FileService
-}
-
-func (h *NLPHandler) CreateJob(c *gin.Context) {
-	api.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "NLP job creation not yet implemented")
-}
-
-func (h *NLPHandler) GetJobStatus(c *gin.Context) {
-	api.Error(c, http.StatusNotFound, "JOB_NOT_FOUND", "NLP job not found")
-}

backend/internal/handler/router.go

@@ -18,7 +18,4 @@ func SetupRoutes(r *gin.Engine, h *Handler) {
 
 	api.POST("/ocr/jobs", h.OCR.CreateJob)
 	api.GET("/ocr/jobs/:id", h.OCR.GetJobStatus)
-
-	api.POST("/nlp/jobs", h.NLP.CreateJob)
-	api.GET("/nlp/jobs/:id", h.NLP.GetJobStatus)
 }

backend/internal/model/file.go

@@ -8,7 +8,6 @@ type File struct {
 	StorageKey string `json:"-"`
 	Checksum   string `json:"-"`
 	OcrText    string `json:"ocrText,omitempty"`
-	NlpData    string `json:"nlpData,omitempty"`
 	CreatedAt  string `json:"createdAt"`
 	UpdatedAt  string `json:"updatedAt"`
 }

backend/internal/nlp/client.go

@@ -1,80 +0,0 @@
-package nlp
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-)
-
-type Client struct {
-	endpoint   string
-	httpClient *http.Client
-}
-
-func NewClient(endpoint string) *Client {
-	return &Client{
-		endpoint: endpoint,
-		httpClient: &http.Client{
-			Timeout: 60 * time.Second,
-		},
-	}
-}
-
-func (c *Client) Analyze(text string) (*AnalyzedFile, error) {
-	reqBody, err := json.Marshal(NLPRequest{Text: text})
-	if err != nil {
-		return nil, fmt.Errorf("marshal request: %w", err)
-	}
-
-	resp, err := c.httpClient.Post(
-		c.endpoint+"/nlp",
-		"application/json",
-		bytes.NewReader(reqBody),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("call nlp server: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("read response: %w", err)
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("nlp server returned %d: %s", resp.StatusCode, string(body))
-	}
-
-	var nlpResp NLPResponse
-	if err := json.Unmarshal(body, &nlpResp); err != nil {
-		return nil, fmt.Errorf("decode response: %w", err)
-	}
-
-	if nlpResp.ErrorCode != 0 {
-		return nil, fmt.Errorf("nlp error %d: %s", nlpResp.ErrorCode, nlpResp.Message)
-	}
-
-	result := &AnalyzedFile{
-		Entities:   nlpResp.Result.Entities,
-		NounChunks: nlpResp.Result.NounChunks,
-		Sentences:  nlpResp.Result.Sentences,
-	}
-
-	return result, nil
-}
-
-func (c *Client) HealthCheck() error {
-	resp, err := c.httpClient.Get(c.endpoint + "/health")
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("health check failed: status %d", resp.StatusCode)
-	}
-	return nil
-}

backend/internal/nlp/types.go

@@ -1,41 +0,0 @@
-package nlp
-
-type NLPRequest struct {
-	Text string `json:"text"`
-}
-
-type NLPResponse struct {
-	ErrorCode int        `json:"errorCode"`
-	Result    NLPResult  `json:"result"`
-	Message   string     `json:"message,omitempty"`
-}
-
-type NLPResult struct {
-	Entities    []NLPEntity  `json:"entities"`
-	Tokens      []NLPToken   `json:"tokens"`
-	NounChunks  []string     `json:"noun_chunks"`
-	Sentences   []string     `json:"sentences"`
-}
-
-type NLPEntity struct {
-	Text       string  `json:"text"`
-	Label      string  `json:"label"`
-	Start      int     `json:"start"`
-	End        int     `json:"end"`
-	Confidence float64 `json:"confidence"`
-}
-
-type NLPToken struct {
-	Text    string `json:"text"`
-	Lemma   string `json:"lemma"`
-	POS     string `json:"pos"`
-	Tag     string `json:"tag"`
-	Dep     string `json:"dep"`
-	IsStop  bool   `json:"is_stop"`
-}
-
-type AnalyzedFile struct {
-	Entities    []NLPEntity `json:"entities"`
-	NounChunks  []string    `json:"noun_chunks"`
-	Sentences   []string    `json:"sentences"`
-}

backend/internal/service/file.go

@@ -113,13 +113,6 @@ func (s *FileService) UpdateOCRText(id, text string) error {
 	})
 }
 
-func (s *FileService) UpdateNLPData(id, nlpData string) error {
-	return s.queries.UpdateNLPData(context.Background(), db.UpdateNLPDataParams{
-		NlpData: nlpData,
-		ID:      id,
-	})
-}
-
 func dbToModel(f db.File) model.File {
 	return model.File{
 		ID:         f.ID,
@@ -129,7 +122,6 @@ func dbToModel(f db.File) model.File {
 		StorageKey: f.StorageKey,
 		Checksum:   f.Checksum,
 		OcrText:    f.OcrText,
-		NlpData:    f.NlpData,
 		CreatedAt:  f.CreatedAt.String(),
 		UpdatedAt:  f.UpdatedAt.String(),
 	}

backend/internal/service/nlp.go

@@ -1,102 +0,0 @@
-package service
-
-import (
-	"encoding/json"
-	"log"
-	"os"
-
-	"github.com/vaultdrop/backend/internal/config"
-	"github.com/vaultdrop/backend/internal/nlp"
-)
-
-type NLPJob struct {
-	FileID   string
-	FilePath string
-	OCRText  string
-}
-
-type NLPService struct {
-	client  *nlp.Client
-	fileSvc *FileService
-	jobs    chan NLPJob
-}
-
-func NewNLPService(cfg *config.Config, fileSvc *FileService) *NLPService {
-	return &NLPService{
-		client:  nlp.NewClient(cfg.NLPEndpoint),
-		fileSvc: fileSvc,
-		jobs:    make(chan NLPJob, 100),
-	}
-}
-
-func (s *NLPService) Start() {
-	go s.worker()
-	log.Println("[NLP] Worker started")
-}
-
-func (s *NLPService) Stop() {
-	close(s.jobs)
-	log.Println("[NLP] Worker stopped")
-}
-
-func (s *NLPService) Enqueue(fileID, filePath, ocrText string) {
-	s.jobs <- NLPJob{FileID: fileID, FilePath: filePath, OCRText: ocrText}
-	log.Printf("[NLP] Enqueued file %s", fileID)
-}
-
-func (s *NLPService) worker() {
-	for job := range s.jobs {
-		s.process(job)
-	}
-}
-
-func (s *NLPService) process(job NLPJob) {
-	log.Printf("[NLP] Processing file %s", job.FileID)
-
-	text := job.OCRText
-
-	if text == "" {
-		data, err := os.ReadFile(job.FilePath)
-		if err != nil {
-			log.Printf("[NLP] Failed to read file %s: %v", job.FileID, err)
-			return
-		}
-		text = string(data)
-	}
-
-	if text == "" {
-		log.Printf("[NLP] No text to analyze for file %s, skipping", job.FileID)
-		return
-	}
-
-	result, err := s.client.Analyze(text)
-	if err != nil {
-		log.Printf("[NLP] Failed to analyze file %s: %v", job.FileID, err)
-		return
-	}
-
-	nlpData, err := json.Marshal(result)
-	if err != nil {
-		log.Printf("[NLP] Failed to marshal NLP result for file %s: %v", job.FileID, err)
-		return
-	}
-
-	if err := s.fileSvc.UpdateNLPData(job.FileID, string(nlpData)); err != nil {
-		log.Printf("[NLP] Failed to update nlp_data for file %s: %v", job.FileID, err)
-		return
-	}
-
-	log.Printf("[NLP] Completed file %s (%d entities, %d sentences)", job.FileID, len(result.Entities), len(result.Sentences))
-}
-
-func (s *NLPService) AnalyzeText(text string) (*nlp.AnalyzedFile, error) {
-	return s.client.Analyze(text)
-}
-
-func (s *NLPService) HealthCheck() error {
-	return s.client.HealthCheck()
-}
-
-func (s *NLPService) QueueLength() int {
-	return len(s.jobs)
-}

backend/internal/service/ocr.go

@@ -17,7 +17,6 @@ type OCRJob struct {
 type OCRService struct {
 	client  *ocr.Client
 	fileSvc *FileService
-	nlpSvc  *NLPService
 	jobs    chan OCRJob
 }
 
@@ -29,10 +28,6 @@ func NewOCRService(cfg *config.Config, fileSvc *FileService) *OCRService {
 	}
 }
 
-func (s *OCRService) SetNLPService(nlpSvc *NLPService) {
-	s.nlpSvc = nlpSvc
-}
-
 func (s *OCRService) Start() {
 	go s.worker()
 	log.Println("[OCR] Worker started")
@@ -77,10 +72,6 @@ func (s *OCRService) process(job OCRJob) {
 	}
 
 	log.Printf("[OCR] Completed file %s (%d chars)", job.FileID, len(text))
-
-	if s.nlpSvc != nil {
-		s.nlpSvc.Enqueue(job.FileID, job.FilePath, text)
-	}
 }
 
 func (s *OCRService) RecognizeFromBytes(data []byte) ([]ocr.TextBlock, error) {

backend/nlp-server/Dockerfile

@@ -1,21 +0,0 @@
-FROM python:3.10-slim
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential curl \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN pip install --no-cache-dir \
-    spacy \
-    fastapi \
-    uvicorn \
-    python-multipart
-
-RUN python -m spacy download fr_core_news_md
-
-COPY server.py /workspace/server.py
-
-WORKDIR /workspace
-
-EXPOSE 8080
-
-CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8080"]

backend/nlp-server/server.py

@@ -1,110 +0,0 @@
-import os
-import logging
-from typing import List, Optional
-
-import spacy
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("nlp-server")
-
-NLP_MODEL = os.getenv("NLP_MODEL", "fr_core_news_md")
-
-print(f"[INIT] Loading spaCy model '{NLP_MODEL}'...", flush=True)
-nlp = spacy.load(NLP_MODEL)
-print("[INIT] spaCy model ready.", flush=True)
-
-app = FastAPI()
-
-
-class NLPRequest(BaseModel):
-    text: str
-
-
-class Entity(BaseModel):
-    text: str
-    label: str
-    start: int
-    end: int
-
-
-class NLPEntityResult(BaseModel):
-    text: str
-    label: str
-    start: int
-    end: int
-    confidence: float
-
-
-class NLPToken(BaseModel):
-    text: str
-    lemma: str
-    pos: str
-    tag: str
-    dep: str
-    is_stop: bool
-
-
-class NLPResult(BaseModel):
-    entities: List[NLPEntityResult]
-    tokens: List[NLPToken]
-    noun_chunks: List[str]
-    sentences: List[str]
-
-
-class NLPResponse(BaseModel):
-    errorCode: int
-    result: Optional[NLPResult] = None
-    message: Optional[str] = None
-
-
-@app.get("/health")
-def health():
-    return {"status": "healthy", "service": "spaCy NLP Server", "model": NLP_MODEL}
-
-
-@app.post("/nlp")
-def analyze(req: NLPRequest):
-    print(f"[NLP] Request received, text length: {len(req.text)}", flush=True)
-    try:
-        doc = nlp(req.text)
-
-        entities = []
-        for ent in doc.ents:
-            entities.append(NLPEntityResult(
-                text=ent.text,
-                label=ent.label_,
-                start=ent.start_char,
-                end=ent.end_char,
-                confidence=0.0,
-            ))
-
-        tokens = []
-        for token in doc:
-            tokens.append(NLPToken(
-                text=token.text,
-                lemma=token.lemma_,
-                pos=token.pos_,
-                tag=token.tag_,
-                dep=token.dep_,
-                is_stop=token.is_stop,
-            ))
-
-        noun_chunks = [chunk.text for chunk in doc.noun_chunks]
-
-        sentences = [sent.text.strip() for sent in doc.sents]
-
-        result = NLPResult(
-            entities=entities,
-            tokens=tokens,
-            noun_chunks=noun_chunks,
-            sentences=sentences,
-        )
-
-        print(f"[NLP] Done: {len(entities)} entities, {len(tokens)} tokens, {len(sentences)} sentences", flush=True)
-        return {"errorCode": 0, "result": result}
-
-    except Exception as e:
-        logger.exception("NLP analysis failed")
-        return {"errorCode": 2, "message": str(e)}

docker-compose.yml

@@ -15,22 +15,6 @@ services:
       retries: 5
       start_period: 120s
 
-  spacy-nlp:
-    build:
-      context: ./backend/nlp-server
-      dockerfile: Dockerfile
-    ports:
-      - "9091:8080"
-    environment:
-      - NLP_MODEL=fr_core_news_md
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-sf", "http://localhost:8080/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 60s
-
   # backend:
   #   build:
   #     context: ./backend
@@ -41,7 +25,6 @@ services:
   #     - PORT=8080
   #     - DB_PATH=/data/vaultdrop.db
   #     - OCR_ENDPOINT=http://paddleocr:8080
-  #     - NLP_ENDPOINT=http://spacy-nlp:8080
   #   volumes:
   #     - backend-data:/data
   #     - ./backend/uploads:/app/uploads