From 15f4a2746d35974e128a6f965a7557fece1555a0 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Fri, 19 Jul 2024 17:26:59 -0700 Subject: [PATCH 1/3] patch CVE in old express-ws pkg --- .github/workflows/dev-image.yaml | 80 +++++++++++++++++++ server/index.js | 2 +- server/package.json | 4 +- .../websocket/websock-branding-collab.js | 2 +- .../websocket/websock-multi-turn-chat.js | 2 +- server/utils/boot/index.js | 11 ++- server/yarn.lock | 22 ++--- 7 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/dev-image.yaml diff --git a/.github/workflows/dev-image.yaml b/.github/workflows/dev-image.yaml new file mode 100644 index 00000000000..4561fd62722 --- /dev/null +++ b/.github/workflows/dev-image.yaml @@ -0,0 +1,80 @@ +name: Publish AnythingLLM Primary Docker image (amd64/arm64) + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ['master'] # master branch only. Do not modify. + paths-ignore: + - '**.md' + - 'cloud-deployments/*' + - 'images/**/*' + - '.vscode/**/*' + - '**/.env.example' + - '.github/ISSUE_TEMPLATE/**/*' + - 'embed/**/*' # Embed should be published to frontend (yarn build:publish) if any changes are introduced + - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images. + - 'docker/vex/*' # CVE exceptions we know are not in risk + +jobs: + push_multi_platform_to_registries: + name: Push Docker multi-platform image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Check if DockerHub build needed + shell: bash + run: | + # Check if the secret for USERNAME is set (don't even check for the password) + if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then + echo "DockerHub build not needed" + echo "enabled=false" >> $GITHUB_OUTPUT + else + echo "DockerHub build needed" + echo "enabled=true" >> $GITHUB_OUTPUT + fi + id: dockerhub + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR + if: steps.dockerhub.outputs.enabled == 'true' + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: | + ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }} + tags: | + type=raw,value=dev + + - name: Build and push multi-platform Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/Dockerfile + push: true + sbom: true + provenance: mode=max + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/server/index.js b/server/index.js index c2f584443cc..757f353449d 100644 --- a/server/index.js +++ b/server/index.js @@ -41,7 +41,7 @@ app.use( if (!!process.env.ENABLE_HTTPS) { bootSSL(app, process.env.SERVER_PORT || 3001); } else { - require("express-ws")(app); // load WebSockets in non-SSL mode. + require("@mintplex-labs/express-ws").default(app); // load WebSockets in non-SSL mode. } app.use("/api", apiRouter); diff --git a/server/package.json b/server/package.json index 57c9256860a..4f07e68b577 100644 --- a/server/package.json +++ b/server/package.json @@ -33,6 +33,7 @@ "@langchain/openai": "0.0.28", "@langchain/textsplitters": "0.0.0", "@mintplex-labs/bree": "^9.2.5", + "@mintplex-labs/express-ws": "^5.0.7", "@pinecone-database/pinecone": "^2.0.1", "@prisma/client": "5.3.1", "@qdrant/js-client-rest": "^1.9.0", @@ -49,7 +50,6 @@ "dotenv": "^16.0.3", "elevenlabs": "^0.5.0", "express": "^4.18.2", - "express-ws": "^5.0.2", "extract-json-from-string": "^1.0.1", "extract-zip": "^2.0.1", "graphql": "^16.7.1", @@ -101,4 +101,4 @@ "nodemon": "^2.0.22", "prettier": "^3.0.3" } -} \ No newline at end of file +} diff --git a/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js b/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js index 7229bd882d9..809aafd3c80 100644 --- a/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js +++ b/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js @@ -13,7 +13,7 @@ const { const path = require("path"); const port = 3000; const app = express(); -require("express-ws")(app); +require("@mintplex-labs/express-ws").default(app); // load WebSockets in non-SSL mode. require("dotenv").config({ path: `../../../../../.env.development` }); // Debugging echo function if this is working for you. diff --git a/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js b/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js index 6277978cc78..6c58709bbbb 100644 --- a/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js +++ b/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js @@ -13,7 +13,7 @@ const { const path = require("path"); const port = 3000; const app = express(); -require("express-ws")(app); +require("@mintplex-labs/express-ws").default(app); // load WebSockets in non-SSL mode. require("dotenv").config({ path: `../../../../../.env.development` }); // Debugging echo function if this is working for you. diff --git a/server/utils/boot/index.js b/server/utils/boot/index.js index 8a3dcbd25aa..979c8e137ca 100644 --- a/server/utils/boot/index.js +++ b/server/utils/boot/index.js @@ -4,6 +4,15 @@ const { EncryptionManager } = require("../EncryptionManager"); const { CommunicationKey } = require("../comKey"); const setupTelemetry = require("../telemetry"); +// Testing SSL? You can make a self signed certificate and point the ENVs to that location +// make a directory in server called 'sslcert' - cd into it +// - openssl genrsa -aes256 -passout pass:gsahdg -out server.pass.key 4096 +// - openssl rsa -passin pass:gsahdg -in server.pass.key -out server.key +// - rm server.pass.key +// - openssl req -new -key server.key -out server.csr +// Update .env keys with the correct values and boot. These are temporary and not real SSL certs - only use for local. +// Test with https://localhost:3001/api/ping +// build and copy frontend to server/public with correct API_BASE and start server in prod model and all should be ok function bootSSL(app, port = 3001) { try { console.log( @@ -26,7 +35,7 @@ function bootSSL(app, port = 3001) { }) .on("error", catchSigTerms); - require("express-ws")(app, server); // Apply same certificate + server for WSS connections + require("@mintplex-labs/express-ws").default(app, server); return { app, server }; } catch (e) { console.error( diff --git a/server/yarn.lock b/server/yarn.lock index 37477622f66..3c5484d4b90 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -704,6 +704,13 @@ p-wait-for "3" safe-timers "^1.1.0" +"@mintplex-labs/express-ws@^5.0.7": + version "5.0.7" + resolved "https://registry.yarnpkg.com/@mintplex-labs/express-ws/-/express-ws-5.0.7.tgz#ef22a849e2c21a8ab93bd68adbe52e278db4d858" + integrity sha512-cwAvW1aB+/nu8QvkCDnLzav9UOAAlRz042PYHh/bS7z4xpucKP2c002+ZbrW6+flA8qIwoBn3FrSjLCHj7Pi2w== + dependencies: + ws "^7.5.10" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -2895,13 +2902,6 @@ expr-eval@^2.0.2: resolved "https://registry.yarnpkg.com/expr-eval/-/expr-eval-2.0.2.tgz#fa6f044a7b0c93fde830954eb9c5b0f7fbc7e201" integrity sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg== -express-ws@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/express-ws/-/express-ws-5.0.2.tgz#5b02d41b937d05199c6c266d7cc931c823bda8eb" - integrity sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ== - dependencies: - ws "^7.4.6" - express@^4.18.2: version "4.19.2" resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" @@ -6823,10 +6823,10 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== -ws@^7.4.6: - version "7.5.9" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" - integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^7.5.10: + version "7.5.10" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" + integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== xtend@^4.0.0: version "4.0.2" From f4587a9eb1f4ef6a0f9a5b7f90bca691748bfc55 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Fri, 19 Jul 2024 17:29:55 -0700 Subject: [PATCH 2/3] patch workflow --- .github/workflows/dev-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-image.yaml b/.github/workflows/dev-image.yaml index 4561fd62722..affc6411af4 100644 --- a/.github/workflows/dev-image.yaml +++ b/.github/workflows/dev-image.yaml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: ['master'] # master branch only. Do not modify. + branches: ['patch-express-ws'] # master branch only. Do not modify. paths-ignore: - '**.md' - 'cloud-deployments/*' From b583fec847dabacd30472e67fd57048199d3ffd1 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Fri, 19 Jul 2024 17:38:50 -0700 Subject: [PATCH 3/3] remove dev-image --- .github/workflows/dev-image.yaml | 80 -------------------------------- 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/dev-image.yaml diff --git a/.github/workflows/dev-image.yaml b/.github/workflows/dev-image.yaml deleted file mode 100644 index affc6411af4..00000000000 --- a/.github/workflows/dev-image.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: Publish AnythingLLM Primary Docker image (amd64/arm64) - -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: ['patch-express-ws'] # master branch only. Do not modify. - paths-ignore: - - '**.md' - - 'cloud-deployments/*' - - 'images/**/*' - - '.vscode/**/*' - - '**/.env.example' - - '.github/ISSUE_TEMPLATE/**/*' - - 'embed/**/*' # Embed should be published to frontend (yarn build:publish) if any changes are introduced - - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images. - - 'docker/vex/*' # CVE exceptions we know are not in risk - -jobs: - push_multi_platform_to_registries: - name: Push Docker multi-platform image to multiple registries - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - - name: Check if DockerHub build needed - shell: bash - run: | - # Check if the secret for USERNAME is set (don't even check for the password) - if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then - echo "DockerHub build not needed" - echo "enabled=false" >> $GITHUB_OUTPUT - else - echo "DockerHub build needed" - echo "enabled=true" >> $GITHUB_OUTPUT - fi - id: dockerhub - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR - if: steps.dockerhub.outputs.enabled == 'true' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: | - ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }} - tags: | - type=raw,value=dev - - - name: Build and push multi-platform Docker image - uses: docker/build-push-action@v6 - with: - context: . - file: ./docker/Dockerfile - push: true - sbom: true - provenance: mode=max - platforms: linux/amd64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max