spacy nlp ner

backend/internal/service/file.go

@@ -113,6 +113,13 @@ func (s *FileService) UpdateOCRText(id, text string) error {
 	})
 }
 
+func (s *FileService) UpdateNLPData(id, nlpData string) error {
+	return s.queries.UpdateNLPData(context.Background(), db.UpdateNLPDataParams{
+		NlpData: nlpData,
+		ID:      id,
+	})
+}
+
 func dbToModel(f db.File) model.File {
 	return model.File{
 		ID:         f.ID,
@@ -122,6 +129,7 @@ func dbToModel(f db.File) model.File {
 		StorageKey: f.StorageKey,
 		Checksum:   f.Checksum,
 		OcrText:    f.OcrText,
+		NlpData:    f.NlpData,
 		CreatedAt:  f.CreatedAt.String(),
 		UpdatedAt:  f.UpdatedAt.String(),
 	}

backend/internal/service/nlp.go

@@ -0,0 +1,102 @@
+package service
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+
+	"github.com/vaultdrop/backend/internal/config"
+	"github.com/vaultdrop/backend/internal/nlp"
+)
+
+type NLPJob struct {
+	FileID   string
+	FilePath string
+	OCRText  string
+}
+
+type NLPService struct {
+	client  *nlp.Client
+	fileSvc *FileService
+	jobs    chan NLPJob
+}
+
+func NewNLPService(cfg *config.Config, fileSvc *FileService) *NLPService {
+	return &NLPService{
+		client:  nlp.NewClient(cfg.NLPEndpoint),
+		fileSvc: fileSvc,
+		jobs:    make(chan NLPJob, 100),
+	}
+}
+
+func (s *NLPService) Start() {
+	go s.worker()
+	log.Println("[NLP] Worker started")
+}
+
+func (s *NLPService) Stop() {
+	close(s.jobs)
+	log.Println("[NLP] Worker stopped")
+}
+
+func (s *NLPService) Enqueue(fileID, filePath, ocrText string) {
+	s.jobs <- NLPJob{FileID: fileID, FilePath: filePath, OCRText: ocrText}
+	log.Printf("[NLP] Enqueued file %s", fileID)
+}
+
+func (s *NLPService) worker() {
+	for job := range s.jobs {
+		s.process(job)
+	}
+}
+
+func (s *NLPService) process(job NLPJob) {
+	log.Printf("[NLP] Processing file %s", job.FileID)
+
+	text := job.OCRText
+
+	if text == "" {
+		data, err := os.ReadFile(job.FilePath)
+		if err != nil {
+			log.Printf("[NLP] Failed to read file %s: %v", job.FileID, err)
+			return
+		}
+		text = string(data)
+	}
+
+	if text == "" {
+		log.Printf("[NLP] No text to analyze for file %s, skipping", job.FileID)
+		return
+	}
+
+	result, err := s.client.Analyze(text)
+	if err != nil {
+		log.Printf("[NLP] Failed to analyze file %s: %v", job.FileID, err)
+		return
+	}
+
+	nlpData, err := json.Marshal(result)
+	if err != nil {
+		log.Printf("[NLP] Failed to marshal NLP result for file %s: %v", job.FileID, err)
+		return
+	}
+
+	if err := s.fileSvc.UpdateNLPData(job.FileID, string(nlpData)); err != nil {
+		log.Printf("[NLP] Failed to update nlp_data for file %s: %v", job.FileID, err)
+		return
+	}
+
+	log.Printf("[NLP] Completed file %s (%d entities, %d sentences)", job.FileID, len(result.Entities), len(result.Sentences))
+}
+
+func (s *NLPService) AnalyzeText(text string) (*nlp.AnalyzedFile, error) {
+	return s.client.Analyze(text)
+}
+
+func (s *NLPService) HealthCheck() error {
+	return s.client.HealthCheck()
+}
+
+func (s *NLPService) QueueLength() int {
+	return len(s.jobs)
+}

backend/internal/service/ocr.go

@@ -17,6 +17,7 @@ type OCRJob struct {
 type OCRService struct {
 	client  *ocr.Client
 	fileSvc *FileService
+	nlpSvc  *NLPService
 	jobs    chan OCRJob
 }
 
@@ -28,6 +29,10 @@ func NewOCRService(cfg *config.Config, fileSvc *FileService) *OCRService {
 	}
 }
 
+func (s *OCRService) SetNLPService(nlpSvc *NLPService) {
+	s.nlpSvc = nlpSvc
+}
+
 func (s *OCRService) Start() {
 	go s.worker()
 	log.Println("[OCR] Worker started")
@@ -72,6 +77,10 @@ func (s *OCRService) process(job OCRJob) {
 	}
 
 	log.Printf("[OCR] Completed file %s (%d chars)", job.FileID, len(text))
+
+	if s.nlpSvc != nil {
+		s.nlpSvc.Enqueue(job.FileID, job.FilePath, text)
+	}
 }
 
 func (s *OCRService) RecognizeFromBytes(data []byte) ([]ocr.TextBlock, error) {