devflowinc · cdxker · Nov 19, 2024 · Nov 8, 2024 · Nov 8, 2024 · Nov 8, 2024
diff --git a/.env.server b/.env.server
@@ -46,4 +46,5 @@ VECTOR_SIZES="384,512,768,1024,1536,3072"
 RUST_LOG="INFO"
 BM25_ACTIVE="true"
 FIRECRAWL_URL=https://api.firecrawl.dev 
-FIRECRAWL_API_KEY=fc-abdef**************
+FIRECRAWL_API_KEY=fc-abdef**************
+PDF2MD_URL="http://localhost:8081"
diff --git a/.github/ISSUE_TEMPLATE/issue-template.md b/.github/ISSUE_TEMPLATE/issue-template.md
@@ -13,11 +13,7 @@ assignees: ''
 
 ### Target(s)
 
-<replace w/ one or more of the following options: `server`, `search`, `chat`>
-
-### Requirement to close
-
-<please describe what is required to close this issue here>
+<replace w/ name of the service(s) which are associated with this issue>
 
 ### Community channels
 

diff --git a/.github/workflows/push-pdf2md-server.yml b/.github/workflows/push-pdf2md-server.yml
@@ -0,0 +1,149 @@
+name: Create PDF2MD Docker Images
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "pdf2md/server/**"
+
+jobs:
+  pdf2md-server:
+    name: Push PDF2MD Server image
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        runner: [blacksmith-8vcpu-ubuntu-2204]
+        platform: [linux/amd64]
+        exclude:
+          - runner: blacksmith-8vcpu-ubuntu-2204
+            platform: linux/arm64
+          - runner: blacksmith-8vcpu-ubuntu-2204-arm
+            platform: linux/amd64
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            trieve/pdf2md-server
+          tags: |
+            type=raw,latest
+            type=sha
+
+      - name: Build and push Docker image
+        uses: useblacksmith/build-push-action@v1.0.0-beta
+        with:
+          platforms: ${{ matrix.platform }}
+          context: pdf2md/
+          file: ./pdf2md/server/Dockerfile.pdf2md-server
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  chunk-worker:
+    name: Push PDF2MD Chunk Worker image
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        runner: [blacksmith-8vcpu-ubuntu-2204]
+        platform: [linux/amd64]
+        exclude:
+          - runner: blacksmith-8vcpu-ubuntu-2204
+            platform: linux/arm64
+          - runner: blacksmith-8vcpu-ubuntu-2204-arm
+            platform: linux/amd64
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            trieve/chunk-worker
+          tags: |
+            type=raw,latest
+            type=sha
+
+      - name: Build and push Docker image
+        uses: useblacksmith/build-push-action@v1.0.0-beta
+        with:
+          platforms: ${{ matrix.platform }}
+          context: pdf2md/
+          file: ./pdf2md/server/Dockerfile.chunk-worker
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  supervisor-worker:
+    name: Push PDF2MD Supervisor Worker image
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        runner: [blacksmith-8vcpu-ubuntu-2204]
+        platform: [linux/amd64]
+        exclude:
+          - runner: blacksmith-8vcpu-ubuntu-2204
+            platform: linux/arm64
+          - runner: blacksmith-8vcpu-ubuntu-2204-arm
+            platform: linux/amd64
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            trieve/supervisor-worker
+          tags: |
+            type=raw,latest
+            type=sha
+
+      - name: Build and push Docker image
+        uses: useblacksmith/build-push-action@v1.0.0-beta
+        with:
+          platforms: ${{ matrix.platform }}
+          context: pdf2md/
+          file: ./pdf2md/server/Dockerfile.supervisor-worker
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.gitignore b/.gitignore
@@ -18,7 +18,7 @@ story_html.zip
 testing.ipynb
 output.json
 temp.json
-analytics/analytics-server/target
+**/target
 server/target
 server/images
 server/tantivy
@@ -92,4 +92,6 @@ server/migrations/2024-07-26-165058_move_config_to_table/down.sql
 server/migrations/2024-07-26-165058_move_config_to_table/up.sql
 dist/**
 
+
 clients/python-sdk/dist
+pdf2md/ch_migrations/chm.toml
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -2,7 +2,11 @@
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
-  "rust-analyzer.linkedProjects": ["./server/Cargo.toml"],
+  "rust-analyzer.linkedProjects": [
+    "./server/Cargo.toml",
+    "./pdf2md/server/Cargo.toml",
+    "./pdf2md/cli/Cargo.toml"
+  ],
   "rust-analyzer.showUnlinkedFileNotification": false,
   "rust-analyzer.server.path": "~/.cargo/bin/rust-analyzer",
   "python.analysis.typeCheckingMode": "basic",

diff --git a/pdf2md/.env.dist b/pdf2md/.env.dist
@@ -0,0 +1,27 @@
+# Redis
+REDIS_URL=redis://:thisredispasswordisverysecureandcomplex@localhost:6379
+REDIS_PASSWORD=thisredispasswordisverysecureandcomplex
+
+# Clickhouse
+CLICKHOUSE_URL=http://localhost:8123
+CLICKHOUSE_DB=default
+CLICKHOUSE_USER=clickhouse
+CLICKHOUSE_PASSWORD=password
+
+# S3
+S3_ENDPOINT=http://127.0.0.1:9000
+S3_ACCESS_KEY=ZaaZZaaZZaaZZaaZZaaZ
+S3_SECRET_KEY=ssssssssssssssssssssTTTTTTTTTTTTTTTTTTTT
+S3_BUCKET=trieve
+
+# S3 dockerfile auto-configuration
+MINIO_ROOT_USER=rootuser
+MINIO_ROOT_PASSWORD=rootpassword
+
+# PDF2MD conversion worker services
+LLM_BASE_URL=https://openrouter.ai/api/v1
+LLM_API_KEY=
+LLM_MODEL=gpt-4o-mini
+
+# PDF2MD HTTP API server
+API_KEY=admin
diff --git a/pdf2md/CONTRIBUTING.md b/pdf2md/CONTRIBUTING.md
@@ -0,0 +1,39 @@
+# Contributing to PDF2MD
+
+## Setup ENV's
+
+```bash
+cd server
+cp .env.dist .env
+```
+
+## Run dep processes
+
+```bash
+docker compose --profile dev up -d
+```
+
+## Run Server + Workers
+
+Strongly recommend using tmux or another multiplex system to handle the different proceses.
+
+```bash
+cargo watch -x run #HTTP server
+cargo run --bin supervisor-worker
+cargo run --bin chunk-worker
+```
+
+## CLI
+
+Make your changes then use the following to run:
+
+```bash
+cd cli
+cargo run -- help #or other command instead of help
+```
+
+## Run tailwindcss server for demo UI
+
+```
+npx tailwindcss -i ./static/in.css -o ./static/output.css --watch
+```