From 2c7da4470c32eedcefe865fbe9135b1a0d4aa9ab Mon Sep 17 00:00:00 2001 From: Francisco Bischoff Date: Wed, 17 Jan 2024 00:00:17 +0000 Subject: [PATCH 1/5] Update Ubuntu base image and improve Dockerfile robustness Upgraded the base Ubuntu image to the latest version to incorporate current security and package updates. Enhanced Dockerfile by introducing pipefail shell option to ensure any step in a pipe sequence failing will fail the entire pipeline, increasing build reliability. Refactored user creation commands for clarity and consistency and removed unnecessary unzip and git installations. Simplified RUN instructions in the Dockerfile to facilitate maintenance and potential readability. Adjusted the execution permission setting of entrypoint and healthcheck scripts to be more concise. Normalized quoting and whitespace within RUN commands and shell scripts for consistency across the codebase. Ensured proper ownership of copied files at build time to prevent potential permission issues during container startup. Shifted to WORKDIR instructions for setting the working directory instead of using 'cd' for better readability and Docker best practices. Resolved missing newline characters at the end of shell scripts which follows common conventions and aids in file concatenation. These changes contribute to the stabilization and optimization of the Docker build process, which should result in more consistent and reliable builds for the AnythingLLM project. --- docker/Dockerfile | 129 +++++++++++++++++++++-------------- docker/docker-entrypoint.sh | 11 +-- docker/docker-healthcheck.sh | 10 +-- 3 files changed, 87 insertions(+), 63 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 595099fb753..a1116b5bcef 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,33 +1,38 @@ # Setup base image -FROM ubuntu:jammy-20230522 AS base +FROM ubuntu:jammy-20230916 AS base +# Build arguments ARG ARG_UID=1000 ARG ARG_GID=1000 FROM base AS build-arm64 RUN echo "Preparing build of AnythingLLM image for arm64 architecture" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID -RUN groupadd -g $ARG_GID anythingllm && \ - useradd -u $ARG_UID -m -d /app -s /bin/bash -g anythingllm anythingllm && \ - mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app +RUN groupadd -g "$ARG_GID" anythingllm && \ + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ + mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ @@ -35,19 +40,20 @@ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable -RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-healthcheck.sh USER anythingllm + WORKDIR /app # Puppeteer does not ship with an ARM86 compatible build for Chromium # so web-scraping would be broken in arm docker containers unless we patch it # by manually installing a compatible chromedriver. RUN echo "Need to patch Puppeteer x Chromium support for ARM86 - installing dep!" && \ - curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ - unzip chrome-linux.zip && \ - rm -rf chrome-linux.zip + curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ + unzip chrome-linux.zip && \ + rm -rf chrome-linux.zip ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV CHROME_PATH=/app/chrome-linux/chrome @@ -61,27 +67,31 @@ RUN echo "Done running arm64 specific installtion steps" FROM base AS build-amd64 RUN echo "Preparing build of AnythingLLM image for non-ARM architecture" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID -RUN groupadd -g $ARG_GID anythingllm && \ - useradd -u $ARG_UID -m -d /app -s /bin/bash -g anythingllm anythingllm && \ - mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app +RUN groupadd -g "$ARG_GID" anythingllm && \ + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ + mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ @@ -89,12 +99,14 @@ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable -RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-healthcheck.sh ############################################# # COMMON BUILD FLOW FOR ALL ARCHS ############################################# + +# hadolint ignore=DL3006 FROM build-${TARGETARCH} AS build RUN echo "Running common build flow of AnythingLLM image for all architectures" @@ -105,39 +117,50 @@ WORKDIR /app FROM build as frontend-deps COPY ./frontend/package.json ./frontend/yarn.lock ./frontend/ -RUN cd ./frontend/ && yarn install --network-timeout 100000 && yarn cache clean +WORKDIR /app/frontend +RUN yarn install --network-timeout 100000 && yarn cache clean +WORKDIR /app # Install server dependencies FROM build as server-deps COPY ./server/package.json ./server/yarn.lock ./server/ -RUN cd ./server/ && yarn install --production --network-timeout 100000 && yarn cache clean +WORKDIR /app/server +RUN yarn install --production --network-timeout 100000 && yarn cache clean +WORKDIR /app # Compile Llama.cpp bindings for node-llama-cpp for this operating system. USER root -RUN cd ./server && npx --no node-llama-cpp download +WORKDIR /app/server +RUN npx --no node-llama-cpp download +WORKDIR /app USER anythingllm # Build the frontend FROM frontend-deps as build-stage COPY ./frontend/ ./frontend/ -RUN cd ./frontend/ && yarn build && yarn cache clean +WORKDIR /app/frontend +RUN yarn build && yarn cache clean +WORKDIR /app # Setup the server FROM server-deps as production-stage COPY --chown=anythingllm:anythingllm ./server/ ./server/ # Copy built static frontend files to the server public directory -COPY --from=build-stage /app/frontend/dist ./server/public +COPY --chown=anythingllm:anythingllm --from=build-stage /app/frontend/dist ./server/public # Copy the collector COPY --chown=anythingllm:anythingllm ./collector/ ./collector/ # Install collector dependencies -RUN cd /app/collector && yarn install --production --network-timeout 100000 && yarn cache clean +WORKDIR /app/collector +RUN yarn install --production --network-timeout 100000 && yarn cache clean # Migrate and Run Prisma against known schema -RUN cd ./server && npx prisma generate --schema=./prisma/schema.prisma -RUN cd ./server && npx prisma migrate deploy --schema=./prisma/schema.prisma +WORKDIR /app/server +RUN npx prisma generate --schema=./prisma/schema.prisma \ + && npx prisma migrate deploy --schema=./prisma/schema.prisma +WORKDIR /app # Setup the environment ENV NODE_ENV=production @@ -151,4 +174,4 @@ HEALTHCHECK --interval=1m --timeout=10s --start-period=1m \ CMD /bin/bash /usr/local/bin/docker-healthcheck.sh || exit 1 # Run the server -ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 3d890db1dee..1ac69e5baf4 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/bash -{ cd /app/server/ &&\ - npx prisma generate --schema=./prisma/schema.prisma &&\ - npx prisma migrate deploy --schema=./prisma/schema.prisma &&\ - node /app/server/index.js +{ + cd /app/server/ && + npx prisma generate --schema=./prisma/schema.prisma && + npx prisma migrate deploy --schema=./prisma/schema.prisma && + node /app/server/index.js } & { node /app/collector/index.js; } & wait -n -exit $? \ No newline at end of file +exit $? diff --git a/docker/docker-healthcheck.sh b/docker/docker-healthcheck.sh index 45a88477d91..49bee3e1bf0 100644 --- a/docker/docker-healthcheck.sh +++ b/docker/docker-healthcheck.sh @@ -4,10 +4,10 @@ response=$(curl --write-out '%{http_code}' --silent --output /dev/null http://localhost:3001/api/ping) # If the HTTP response code is 200 (OK), the server is up -if [ $response -eq 200 ]; then - echo "Server is up" - exit 0 +if [ "$response" -eq 200 ]; then + echo "Server is up" + exit 0 else - echo "Server is down" - exit 1 + echo "Server is down" + exit 1 fi From ec495362b3610796483559339dc6baf2e637d6ef Mon Sep 17 00:00:00 2001 From: Francisco Bischoff Date: Thu, 18 Jan 2024 22:55:47 +0000 Subject: [PATCH 2/5] Add unzip to Docker image dependencies Needed for the arm64 build --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a1116b5bcef..92de13895e0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl gnupg libgfortran5 libgbm1 tzdata netcat \ + unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ From cfcb883412e41ab45efd26c949086ca81d026565 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Fri, 26 Jan 2024 15:48:21 -0800 Subject: [PATCH 3/5] reset tabs --- docker/Dockerfile | 75 +++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 92de13895e0..7ef073db659 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,21 +13,21 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies # hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ @@ -40,20 +40,19 @@ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable -RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-healthcheck.sh USER anythingllm - WORKDIR /app # Puppeteer does not ship with an ARM86 compatible build for Chromium # so web-scraping would be broken in arm docker containers unless we patch it # by manually installing a compatible chromedriver. RUN echo "Need to patch Puppeteer x Chromium support for ARM86 - installing dep!" && \ - curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ - unzip chrome-linux.zip && \ - rm -rf chrome-linux.zip + curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ + unzip chrome-linux.zip && \ + rm -rf chrome-linux.zip ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV CHROME_PATH=/app/chrome-linux/chrome @@ -72,21 +71,21 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies # hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ @@ -99,8 +98,8 @@ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable -RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-healthcheck.sh ############################################# # COMMON BUILD FLOW FOR ALL ARCHS From 812c1591cbd98c9dbd1b9f7bec11561596b018ce Mon Sep 17 00:00:00 2001 From: Francisco Bischoff Date: Mon, 19 Feb 2024 05:30:08 +0000 Subject: [PATCH 4/5] formalized lint rules for hadolint. however the Docker formatting is being handled by MS Docker extension which doesn't indent code as expected. WIP. --- .hadolint.yaml | 8 ++++++ docker/Dockerfile | 72 +++++++++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 .hadolint.yaml diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 00000000000..b76a5107288 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,8 @@ +failure-threshold: warning +ignored: + - DL3008 + - DL3013 +format: tty +trustedRegistries: + - docker.io + - gcr.io diff --git a/docker/Dockerfile b/docker/Dockerfile index 7ef073db659..2433ebdadfc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,23 +11,22 @@ RUN echo "Preparing build of AnythingLLM image for arm64 architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies -# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ @@ -41,7 +40,7 @@ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh + chmod +x /usr/local/bin/docker-healthcheck.sh USER anythingllm WORKDIR /app @@ -50,9 +49,9 @@ WORKDIR /app # so web-scraping would be broken in arm docker containers unless we patch it # by manually installing a compatible chromedriver. RUN echo "Need to patch Puppeteer x Chromium support for ARM86 - installing dep!" && \ - curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ - unzip chrome-linux.zip && \ - rm -rf chrome-linux.zip + curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ + unzip chrome-linux.zip && \ + rm -rf chrome-linux.zip ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV CHROME_PATH=/app/chrome-linux/chrome @@ -69,23 +68,22 @@ RUN echo "Preparing build of AnythingLLM image for non-ARM architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies -# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ @@ -99,7 +97,7 @@ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh + chmod +x /usr/local/bin/docker-healthcheck.sh ############################################# # COMMON BUILD FLOW FOR ALL ARCHS From 8050e36148e6be48c9b9eaa5c5e00c8f865aab33 Mon Sep 17 00:00:00 2001 From: Francisco Bischoff Date: Tue, 20 Feb 2024 02:03:51 +0000 Subject: [PATCH 5/5] found a workaround to keep formatting --- .vscode/settings.json | 5 ++- docker/Dockerfile | 92 ++++++++++++++++++++++--------------------- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ac8c94729a8..a089c904e9d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,5 +14,6 @@ "Weaviate", "Zilliz" ], - "eslint.experimental.useFlatConfig": true -} \ No newline at end of file + "eslint.experimental.useFlatConfig": true, + "docker.languageserver.formatter.ignoreMultilineInstructions": true +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 2433ebdadfc..de25d598b79 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,27 +11,28 @@ RUN echo "Preparing build of AnythingLLM image for arm64 architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ - useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ - mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ + mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ @@ -40,7 +41,7 @@ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh + chmod +x /usr/local/bin/docker-healthcheck.sh USER anythingllm WORKDIR /app @@ -49,9 +50,9 @@ WORKDIR /app # so web-scraping would be broken in arm docker containers unless we patch it # by manually installing a compatible chromedriver. RUN echo "Need to patch Puppeteer x Chromium support for ARM86 - installing dep!" && \ - curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ - unzip chrome-linux.zip && \ - rm -rf chrome-linux.zip + curl https://playwright.azureedge.net/builds/chromium/1088/chromium-linux-arm64.zip -o chrome-linux.zip && \ + unzip chrome-linux.zip && \ + rm -rf chrome-linux.zip ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV CHROME_PATH=/app/chrome-linux/chrome @@ -68,27 +69,28 @@ RUN echo "Preparing build of AnythingLLM image for non-ARM architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl gnupg libgfortran5 libgbm1 tzdata netcat \ - libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ - libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ - libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ - libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ - xdg-utils git build-essential ffmpeg && \ - mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends nodejs && \ - curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ - && dpkg -i yarn_1.22.19_all.deb \ - && rm yarn_1.22.19_all.deb + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + curl gnupg libgfortran5 libgbm1 tzdata netcat \ + libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ + libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ + libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ + xdg-utils git build-essential ffmpeg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends nodejs && \ + curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ + && dpkg -i yarn_1.22.19_all.deb \ + && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID RUN groupadd -g "$ARG_GID" anythingllm && \ - useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ - mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ + mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ @@ -97,7 +99,7 @@ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ - chmod +x /usr/local/bin/docker-healthcheck.sh + chmod +x /usr/local/bin/docker-healthcheck.sh ############################################# # COMMON BUILD FLOW FOR ALL ARCHS @@ -111,7 +113,7 @@ USER anythingllm WORKDIR /app # Install frontend dependencies -FROM build as frontend-deps +FROM build AS frontend-deps COPY ./frontend/package.json ./frontend/yarn.lock ./frontend/ WORKDIR /app/frontend @@ -119,7 +121,7 @@ RUN yarn install --network-timeout 100000 && yarn cache clean WORKDIR /app # Install server dependencies -FROM build as server-deps +FROM build AS server-deps COPY ./server/package.json ./server/yarn.lock ./server/ WORKDIR /app/server RUN yarn install --production --network-timeout 100000 && yarn cache clean @@ -133,14 +135,14 @@ WORKDIR /app USER anythingllm # Build the frontend -FROM frontend-deps as build-stage +FROM frontend-deps AS build-stage COPY ./frontend/ ./frontend/ WORKDIR /app/frontend RUN yarn build && yarn cache clean WORKDIR /app # Setup the server -FROM server-deps as production-stage +FROM server-deps AS production-stage COPY --chown=anythingllm:anythingllm ./server/ ./server/ # Copy built static frontend files to the server public directory @@ -155,8 +157,8 @@ RUN yarn install --production --network-timeout 100000 && yarn cache clean # Migrate and Run Prisma against known schema WORKDIR /app/server -RUN npx prisma generate --schema=./prisma/schema.prisma \ - && npx prisma migrate deploy --schema=./prisma/schema.prisma +RUN npx prisma generate --schema=./prisma/schema.prisma && \ + npx prisma migrate deploy --schema=./prisma/schema.prisma WORKDIR /app # Setup the environment