Initial Services

2026-02-21 22:34:40 +00:00
parent 060629ca94
commit 72c22c2396
2 changed files with 91 additions and 9 deletions
@@ -19,18 +19,42 @@
 - [x] Both sync directions tested and working
 - [x] Old JingTian-Rclone repo archived on Gitea

-## Phase 1: Foundation
+## Phase 1: Services (Services-First Approach)

- [ ] Init Go module
- [ ] SQLite schema + migrations (documents, extractions, processing_log)
+Rationale: De-risk unknowns (Docling OCR quality, Ollama CPU inference speed) before building orchestrator.
+
+### 1a. Docker Compose + Docling
+
+- [ ] Docker Compose base (Docling service)
+- [ ] Test Docling API with sample PDFs (curl)
+- [ ] Validate OCR quality on real PDFs + generated samples
+
+### 1b. Ollama
+
+- [ ] Add Ollama to Docker Compose (Qwen3-1.7B, CPU mode)
+- [ ] Test structured extraction prompt (curl)
+- [ ] Measure inference time per document
+- [ ] Validate extraction accuracy (deadline, doc type, etc.)
+
+### 1c. Python Tools Service
+
+- [ ] FastAPI + openpyxl service
+- [ ] Endpoints: read tracker, write/update rows, get column schema
+- [ ] Test with sample Tracker.xlsx
+
+### 1d. Manual Integration Test
+
+- [ ] Chain test: PDF → Docling → Ollama → Tools → Excel updated
+- [ ] Document any issues / adjustments needed
+
+## Phase 2: Go Orchestrator
+
+- [ ] Init Go module in Tracker/
 - [ ] Config loading (YAML)
- [ ] Docker Compose (Docling, Ollama, Python Tools, Go server)
-
-## Phase 2: Pipeline Core
-
+- [ ] SQLite schema + migrations (documents, extractions, processing_log)
 - [ ] Docling HTTP client
- [ ] Ollama HTTP client + structured extraction prompt
- [ ] Tools service (FastAPI + openpyxl Excel endpoints)
+- [ ] Ollama HTTP client
+- [ ] Tools service HTTP client
 - [ ] File watcher (inotify on Ubuntu, 2-min stability check via SHA256 hash)
 - [ ] State tracker integration (new/changed/skip logic)
 - [ ] Pipeline orchestrator (read latest Tracker before writing, merge not overwrite)
@@ -0,0 +1,58 @@
+services:
+    # Docling
+    docling:
+        # https://github.com/docling-project/docling-serve
+        # 5001: Web UI + API
+
+        image: ghcr.io/docling-project/docling-serve-cpu:v1.13.0
+        container_name: jt-docling
+        restart: unless-stopped
+        environment:
+            DOCLING_SERVE_ENABLE_UI: "true"
+        ports:
+            - "5001:5001"
+        healthcheck:
+            test: ["CMD", "curl", "-sf", "http://localhost:5001/health"]
+            interval: 30s
+            timeout: 10s
+            retries: 3
+            start_period: 60s
+
+    # Ollama
+    ollama:
+        # https://hub.docker.com/r/ollama/ollama
+        # 11434: API port
+        # Model: qwen3:1.7b
+
+        image: ollama/ollama:0.9.3
+        container_name: jt-ollama
+        restart: unless-stopped
+        volumes:
+            - ollama_data:/root/.ollama
+        ports:
+            - "11434:11434"
+        healthcheck:
+            test: ["CMD", "curl", "-sf", "http://localhost:11434/api/tags"]
+            interval: 30s
+            timeout: 10s
+            retries: 5
+            start_period: 30s
+
+    # Tools (Python FastAPI - Excel operations)
+    # tools:
+    #     build: ./Tools/Service
+    #     container_name: jt-tools
+    #     restart: unless-stopped
+    #     volumes:
+    #         - /data/jingtian/BenjaminTeam:/data:rw
+    #     ports:
+    #         - "8080:8080"
+    #     healthcheck:
+    #         test: ["CMD", "curl", "-sf", "http://localhost:8080/health"]
+    #         interval: 30s
+    #         timeout: 10s
+    #         retries: 3
+
+volumes:
+    ollama_data:
+        name: jt-ollama-data