Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove npl

Browse Source
This commit is contained in:
m
2026-07-12 22:15:23 +02:00
parent 010cb2cb49
commit 3650b7ba9d
21 changed files with 8 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/config/config.go
-2
View File
@@ -6,7 +6,6 @@ type Config struct {
Port string
DBPath string
OCREndpoint string
NLPEndpoint string
UploadDir string
HMACSecret string
ServerHost string
@@ -17,7 +16,6 @@ func Load() *Config {
Port: envOr("PORT", "8080"),
DBPath: envOr("DB_PATH", "vaultdrop.db"),
OCREndpoint: envOr("OCR_ENDPOINT", "http://localhost:9090"),
NLPEndpoint: envOr("NLP_ENDPOINT", "http://localhost:9091"),
UploadDir: envOr("UPLOAD_DIR", "./uploads"),
HMACSecret: envOr("HMAC_SECRET", "thisismyrandomstring"),
ServerHost: envOr("SERVER_HOST", "http://192.168.1.17:8080"),
backend/internal/db/files.sql.go
+4 -24
View File
@@ -12,7 +12,7 @@ import (
const createFile = `-- name: CreateFile :one
INSERT INTO files (id, name, mime_type, size, storage_key, checksum, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data
RETURNING id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at
`
type CreateFileParams struct {
@@ -44,7 +44,6 @@ func (q *Queries) CreateFile(ctx context.Context, arg CreateFileParams) (File, e
&i.OcrText,
&i.CreatedAt,
&i.UpdatedAt,
&i.NlpData,
)
return i, err
}
@@ -60,7 +59,7 @@ func (q *Queries) DeleteFile(ctx context.Context, id string) error {
}
const getFile = `-- name: GetFile :one
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
WHERE id = ? LIMIT 1
`
@@ -77,13 +76,12 @@ func (q *Queries) GetFile(ctx context.Context, id string) (File, error) {
&i.OcrText,
&i.CreatedAt,
&i.UpdatedAt,
&i.NlpData,
)
return i, err
}
const listFiles = `-- name: ListFiles :many
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
ORDER BY created_at DESC
`
@@ -106,7 +104,6 @@ func (q *Queries) ListFiles(ctx context.Context) ([]File, error) {
&i.OcrText,
&i.CreatedAt,
&i.UpdatedAt,
&i.NlpData,
); err != nil {
return nil, err
}
@@ -122,7 +119,7 @@ func (q *Queries) ListFiles(ctx context.Context) ([]File, error) {
}
const listFilesByID = `-- name: ListFilesByID :many
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at, nlp_data FROM files
SELECT id, name, mime_type, size, storage_key, checksum, ocr_text, created_at, updated_at FROM files
WHERE id IN (SELECT value FROM json_each(?))
ORDER BY created_at DESC
`
@@ -146,7 +143,6 @@ func (q *Queries) ListFilesByID(ctx context.Context, jsonEach interface{}) ([]Fi
&i.OcrText,
&i.CreatedAt,
&i.UpdatedAt,
&i.NlpData,
); err != nil {
return nil, err
}
@@ -183,19 +179,3 @@ func (q *Queries) UpdateFile(ctx context.Context, arg UpdateFileParams) error {
)
return err
}
const updateNLPData = `-- name: UpdateNLPData :exec
UPDATE files
SET nlp_data = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?
`
type UpdateNLPDataParams struct {
NlpData string `json:"nlp_data"`
ID string `json:"id"`
}
func (q *Queries) UpdateNLPData(ctx context.Context, arg UpdateNLPDataParams) error {
_, err := q.db.ExecContext(ctx, updateNLPData, arg.NlpData, arg.ID)
return err
}
backend/internal/db/migrations/003_nlp_data.down.sql
-1
View File
@@ -1 +0,0 @@
ALTER TABLE files DROP COLUMN nlp_data;
backend/internal/db/migrations/003_nlp_data.up.sql
-1
View File
@@ -1 +0,0 @@
ALTER TABLE files ADD COLUMN nlp_data TEXT NOT NULL DEFAULT '';
backend/internal/db/models.go
-1
View File
@@ -18,5 +18,4 @@ type File struct {
OcrText string `json:"ocr_text"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
NlpData string `json:"nlp_data"`
}
backend/internal/db/queries/files.sql
-5
View File
@@ -24,8 +24,3 @@ WHERE id = ?;
-- name: DeleteFile :exec
DELETE FROM files
WHERE id = ?;
-- name: UpdateNLPData :exec
UPDATE files
SET nlp_data = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?;
backend/internal/handler/files.go
-3
View File
@@ -64,7 +64,6 @@ func (h *FileHandler) List(c *gin.Context) {
CreatedAt string `json:"createdAt"`
MimeType string `json:"mimeType"`
OcrText string `json:"ocrText,omitempty"`
NlpData string `json:"nlpData,omitempty"`
UpdatedAt string `json:"updatedAt"`
}
@@ -79,7 +78,6 @@ func (h *FileHandler) List(c *gin.Context) {
Tags: []string{},
CreatedAt: f.CreatedAt,
OcrText: f.OcrText,
NlpData: f.NlpData,
UpdatedAt: f.UpdatedAt,
MimeType: f.MimeType,
}
@@ -124,7 +122,6 @@ func (h *FileHandler) Get(c *gin.Context) {
"createdAt": file.CreatedAt,
"updatedAt": file.UpdatedAt,
"ocrText": file.OcrText,
"nlpData": file.NlpData,
})
}
backend/internal/handler/handler.go
+2 -4
View File
@@ -7,15 +7,13 @@ import (
type Handler struct {
File *FileHandler
OCR *OCRHandler
NLP *NLPHandler
Health *HealthHandler
}
func New(fileSvc *service.FileService, ocrSvc *service.OCRService, nlpSvc *service.NLPService, urlSvc *service.URLService) *Handler {
func New(fileSvc *service.FileService, ocrSvc *service.OCRService, urlSvc *service.URLService) *Handler {
return &Handler{
File: &FileHandler{files: fileSvc, urls: urlSvc, ocr: ocrSvc},
OCR: &OCRHandler{ocr: ocrSvc, files: fileSvc},
NLP: &NLPHandler{nlp: nlpSvc, files: fileSvc},
Health: &HealthHandler{ocr: ocrSvc, nlp: nlpSvc},
Health: &HealthHandler{ocr: ocrSvc},
}
}
backend/internal/handler/health.go
+1 -16
View File
@@ -7,23 +7,8 @@ import (
type HealthHandler struct {
ocr interface{ HealthCheck() error }
nlp interface{ HealthCheck() error }
}
func (h *HealthHandler) Check(c *gin.Context) {
status := gin.H{"status": "healthy"}
if err := h.ocr.HealthCheck(); err != nil {
status["ocr"] = err.Error()
} else {
status["ocr"] = "healthy"
}
if err := h.nlp.HealthCheck(); err != nil {
status["nlp"] = err.Error()
} else {
status["nlp"] = "healthy"
}
api.Success(c, status)
api.Success(c, gin.H{"status": "healthy"})
}
backend/internal/handler/nlp.go
-22
View File
@@ -1,22 +0,0 @@
package handler
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/vaultdrop/backend/internal/service"
"github.com/vaultdrop/backend/pkg/api"
)
type NLPHandler struct {
nlp *service.NLPService
files *service.FileService
}
func (h *NLPHandler) CreateJob(c *gin.Context) {
api.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "NLP job creation not yet implemented")
}
func (h *NLPHandler) GetJobStatus(c *gin.Context) {
api.Error(c, http.StatusNotFound, "JOB_NOT_FOUND", "NLP job not found")
}
backend/internal/handler/router.go
-3
View File
@@ -18,7 +18,4 @@ func SetupRoutes(r *gin.Engine, h *Handler) {
api.POST("/ocr/jobs", h.OCR.CreateJob)
api.GET("/ocr/jobs/:id", h.OCR.GetJobStatus)
api.POST("/nlp/jobs", h.NLP.CreateJob)
api.GET("/nlp/jobs/:id", h.NLP.GetJobStatus)
}
backend/internal/model/file.go
-1
View File
@@ -8,7 +8,6 @@ type File struct {
StorageKey string `json:"-"`
Checksum string `json:"-"`
OcrText string `json:"ocrText,omitempty"`
NlpData string `json:"nlpData,omitempty"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
}
backend/internal/nlp/client.go
-80
View File
@@ -1,80 +0,0 @@
package nlp
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
type Client struct {
endpoint string
httpClient *http.Client
}
func NewClient(endpoint string) *Client {
return &Client{
endpoint: endpoint,
httpClient: &http.Client{
Timeout: 60 * time.Second,
},
}
}
func (c *Client) Analyze(text string) (*AnalyzedFile, error) {
reqBody, err := json.Marshal(NLPRequest{Text: text})
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
resp, err := c.httpClient.Post(
c.endpoint+"/nlp",
"application/json",
bytes.NewReader(reqBody),
)
if err != nil {
return nil, fmt.Errorf("call nlp server: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("nlp server returned %d: %s", resp.StatusCode, string(body))
}
var nlpResp NLPResponse
if err := json.Unmarshal(body, &nlpResp); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
if nlpResp.ErrorCode != 0 {
return nil, fmt.Errorf("nlp error %d: %s", nlpResp.ErrorCode, nlpResp.Message)
}
result := &AnalyzedFile{
Entities: nlpResp.Result.Entities,
NounChunks: nlpResp.Result.NounChunks,
Sentences: nlpResp.Result.Sentences,
}
return result, nil
}
func (c *Client) HealthCheck() error {
resp, err := c.httpClient.Get(c.endpoint + "/health")
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("health check failed: status %d", resp.StatusCode)
}
return nil
}
backend/internal/nlp/types.go
-41
View File
@@ -1,41 +0,0 @@
package nlp
type NLPRequest struct {
Text string `json:"text"`
}
type NLPResponse struct {
ErrorCode int `json:"errorCode"`
Result NLPResult `json:"result"`
Message string `json:"message,omitempty"`
}
type NLPResult struct {
Entities []NLPEntity `json:"entities"`
Tokens []NLPToken `json:"tokens"`
NounChunks []string `json:"noun_chunks"`
Sentences []string `json:"sentences"`
}
type NLPEntity struct {
Text string `json:"text"`
Label string `json:"label"`
Start int `json:"start"`
End int `json:"end"`
Confidence float64 `json:"confidence"`
}
type NLPToken struct {
Text string `json:"text"`
Lemma string `json:"lemma"`
POS string `json:"pos"`
Tag string `json:"tag"`
Dep string `json:"dep"`
IsStop bool `json:"is_stop"`
}
type AnalyzedFile struct {
Entities []NLPEntity `json:"entities"`
NounChunks []string `json:"noun_chunks"`
Sentences []string `json:"sentences"`
}
backend/internal/service/file.go
-8
View File
@@ -113,13 +113,6 @@ func (s *FileService) UpdateOCRText(id, text string) error {
})
}
func (s *FileService) UpdateNLPData(id, nlpData string) error {
return s.queries.UpdateNLPData(context.Background(), db.UpdateNLPDataParams{
NlpData: nlpData,
ID: id,
})
}
func dbToModel(f db.File) model.File {
return model.File{
ID: f.ID,
@@ -129,7 +122,6 @@ func dbToModel(f db.File) model.File {
StorageKey: f.StorageKey,
Checksum: f.Checksum,
OcrText: f.OcrText,
NlpData: f.NlpData,
CreatedAt: f.CreatedAt.String(),
UpdatedAt: f.UpdatedAt.String(),
}
backend/internal/service/nlp.go
-102
View File
@@ -1,102 +0,0 @@
package service
import (
"encoding/json"
"log"
"os"
"github.com/vaultdrop/backend/internal/config"
"github.com/vaultdrop/backend/internal/nlp"
)
type NLPJob struct {
FileID string
FilePath string
OCRText string
}
type NLPService struct {
client *nlp.Client
fileSvc *FileService
jobs chan NLPJob
}
func NewNLPService(cfg *config.Config, fileSvc *FileService) *NLPService {
return &NLPService{
client: nlp.NewClient(cfg.NLPEndpoint),
fileSvc: fileSvc,
jobs: make(chan NLPJob, 100),
}
}
func (s *NLPService) Start() {
go s.worker()
log.Println("[NLP] Worker started")
}
func (s *NLPService) Stop() {
close(s.jobs)
log.Println("[NLP] Worker stopped")
}
func (s *NLPService) Enqueue(fileID, filePath, ocrText string) {
s.jobs <- NLPJob{FileID: fileID, FilePath: filePath, OCRText: ocrText}
log.Printf("[NLP] Enqueued file %s", fileID)
}
func (s *NLPService) worker() {
for job := range s.jobs {
s.process(job)
}
}
func (s *NLPService) process(job NLPJob) {
log.Printf("[NLP] Processing file %s", job.FileID)
text := job.OCRText
if text == "" {
data, err := os.ReadFile(job.FilePath)
if err != nil {
log.Printf("[NLP] Failed to read file %s: %v", job.FileID, err)
return
}
text = string(data)
}
if text == "" {
log.Printf("[NLP] No text to analyze for file %s, skipping", job.FileID)
return
}
result, err := s.client.Analyze(text)
if err != nil {
log.Printf("[NLP] Failed to analyze file %s: %v", job.FileID, err)
return
}
nlpData, err := json.Marshal(result)
if err != nil {
log.Printf("[NLP] Failed to marshal NLP result for file %s: %v", job.FileID, err)
return
}
if err := s.fileSvc.UpdateNLPData(job.FileID, string(nlpData)); err != nil {
log.Printf("[NLP] Failed to update nlp_data for file %s: %v", job.FileID, err)
return
}
log.Printf("[NLP] Completed file %s (%d entities, %d sentences)", job.FileID, len(result.Entities), len(result.Sentences))
}
func (s *NLPService) AnalyzeText(text string) (*nlp.AnalyzedFile, error) {
return s.client.Analyze(text)
}
func (s *NLPService) HealthCheck() error {
return s.client.HealthCheck()
}
func (s *NLPService) QueueLength() int {
return len(s.jobs)
}
backend/internal/service/ocr.go
-9
View File
@@ -17,7 +17,6 @@ type OCRJob struct {
type OCRService struct {
client *ocr.Client
fileSvc *FileService
nlpSvc *NLPService
jobs chan OCRJob
}
@@ -29,10 +28,6 @@ func NewOCRService(cfg *config.Config, fileSvc *FileService) *OCRService {
}
}
func (s *OCRService) SetNLPService(nlpSvc *NLPService) {
s.nlpSvc = nlpSvc
}
func (s *OCRService) Start() {
go s.worker()
log.Println("[OCR] Worker started")
@@ -77,10 +72,6 @@ func (s *OCRService) process(job OCRJob) {
}
log.Printf("[OCR] Completed file %s (%d chars)", job.FileID, len(text))
if s.nlpSvc != nil {
s.nlpSvc.Enqueue(job.FileID, job.FilePath, text)
}
}
func (s *OCRService) RecognizeFromBytes(data []byte) ([]ocr.TextBlock, error) {