ci: updated dockerfiles

.env.local

@@ -0,0 +1,2 @@
+CRAWLAB_TAG=develop
+CRAWLAB_PORT=8082

.github/workflows/docker-crawlab.yml

@@ -268,6 +268,8 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
+          build-args: |
+            CRAWLAB_TAG=${{ needs.setup.outputs.version }}
           push: true
           tags: |
             ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }}

Dockerfile

@@ -1,8 +1,10 @@
-FROM crawlabteam/crawlab-backend:latest AS backend-build
+ARG CRAWLAB_TAG=latest
 
-FROM crawlabteam/crawlab-frontend:latest AS frontend-build
+FROM crawlabteam/crawlab-backend:${CRAWLAB_TAG} AS backend-build
 
-FROM crawlabteam/crawlab-base:latest
+FROM crawlabteam/crawlab-frontend:${CRAWLAB_TAG} AS frontend-build
+
+FROM crawlabteam/crawlab-base:${CRAWLAB_TAG}
 
 # copy backend files
 RUN mkdir -p /opt/bin

backend/.dockerignore

@@ -0,0 +1,41 @@
+# Go build artifacts
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+*.test
+*.out
+
+# Go workspace file
+go.work
+
+# IDE directories
+.idea/
+.vscode/
+
+# Dependencies
+vendor/
+
+# Build output
+bin/
+dist/
+
+# Debug files
+debug/
+
+# Environment files
+.env
+*.env
+
+# System files
+.DS_Store
+Thumbs.db
+
+# Test coverage
+coverage.txt
+*.cover
+
+# Temporary files
+*.tmp
+*~

docker-compose.dev.yml

@@ -0,0 +1,58 @@
+version: '3.3'
+services:
+  backend:
+    build:
+      context: ./backend
+    image: crawlabteam/crawlab-backend:${CRAWLAB_TAG:-develop}
+    env_file:
+      - .env.local
+
+  frontend:
+    build:
+      context: ./frontend
+    image: crawlabteam/crawlab-frontend:${CRAWLAB_TAG:-develop}
+    env_file:
+      - .env.local
+
+  base-image:
+    build:
+      context: ./docker/base-image
+    image: crawlabteam/crawlab-base:${CRAWLAB_TAG:-develop}
+    env_file:
+      - .env.local
+
+  crawlab:
+    build:
+      context: .
+    image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop}
+    depends_on:
+      - backend
+      - frontend
+      - base-image
+    env_file:
+      - .env.local
+
+  master:
+    image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop}
+    environment:
+      CRAWLAB_NODE_MASTER: "Y"
+      CRAWLAB_MONGO_HOST: "mongo"
+    ports:
+      - "${CRAWLAB_PORT:-8082}:8080"
+    depends_on:
+      - mongo
+    env_file:
+      - .env.local
+
+  worker:
+    image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop}
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_MASTER_HOST: "master"
+    depends_on:
+      - master
+    env_file:
+      - .env.local
+
+  mongo:
+    image: mongo:5

docker/base-image/Dockerfile

@@ -22,9 +22,6 @@ RUN bash /app/install/node/node.sh
 # install java
 #RUN bash /app/install/java/java.sh
 
-# install seaweedfs
-RUN bash /app/install/seaweedfs/seaweedfs.sh
-
 # install chromedriver
 RUN bash /app/install/chromedriver/chromedriver.sh
 

docker/base-image/install/chromedriver/chromedriver.sh

@@ -1,21 +1,25 @@
 #!/bin/bash
 
 # version
-version="106.0.5249.61"
+version="stable"
 
 # deps
-apt-get install -y unzip xvfb libxi6 libgconf-2-4
+apt-get install -y xvfb libxi6 libgconf-2-4
 
-# chrome
-wget -q "http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${version}-1_amd64.deb"
-apt-get -y install "./google-chrome-stable_${version}-1_amd64.deb"
-echo `google-chrome --version`
-rm -f "./google-chrome-stable_${version}-1_amd64.deb"
+# install chrome
+npx @puppeteer/browsers install chrome@${version}
 
+# verify chrome version
+if [[ ! "$(google-chrome --version)" =~ ^Google\ Chrome\ ${version} ]]; then
+  echo "ERROR: chrome version does not match. expected: \"Google Chrome ${version}\", but actual is \"$(google-chrome --version)\""
+  exit 1
+fi
 
-# chromedriver
-wget "https://chromedriver.storage.googleapis.com/${version}/chromedriver_linux64.zip"
-unzip chromedriver_linux64.zip
-mv chromedriver /usr/local/bin/chromedriver
-chown root:root /usr/local/bin/chromedriver
-chmod +x /usr/local/bin/chromedriver
+# install chromedriver
+npx @puppeteer/browsers install chromedriver@${version}
+
+# verify chromedriver version
+if [[ ! "$(chromedriver --version)" =~ ^ChromeDriver\ ${version} ]]; then
+  echo "ERROR: chromedriver version does not match. expected: \"ChromeDriver ${version}\", but actual is \"$(chromedriver --version)\""
+  exit 1
+fi

docker/base-image/install/node/node.sh

@@ -1,9 +1,25 @@
 #!/bin/bash
 
-# install node
-curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh \
-  && bash /tmp/nodesource_setup.sh \
-  && apt-get install -y nodejs
+# installs nvm (Node Version Manager)
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash
+
+# download and install Node.js (you may need to restart the terminal)
+nvm install 22
+
+# set node version
+nvm use 22
+
+# verifies the right Node.js version is in the environment
+if [[ ! "$(node -v)" =~ ^v22 ]]; then
+  echo "Node.js version is not v22.x"
+  exit 1
+fi
+
+# verifies the right npm version is in the environment
+if [[ ! "$(npm -v)" =~ ^10 ]]; then
+  echo "npm version is not 10.x"
+  exit 1
+fi
 
 # install node dependencies
 npm install -g \

docker/base-image/install/python/python.sh

@@ -1,11 +1,22 @@
 #!/bin/bash
 
-# install python
-apt-get update \
-	&& apt install software-properties-common -y \
-	&& add-apt-repository ppa:deadsnakes/ppa -y \
-	&& apt install python3.10 -y \
-	&& curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+# install pyenv
+curl https://pyenv.run | bash
+
+# add pyenv to path
+echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
+echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
+echo 'eval "$(pyenv init -)"' >> ~/.bashrc
+source ~/.bashrc
+
+# install python build dependencies
+apt-get install -y make build-essential libssl-dev zlib1g-dev \
+libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+# install python 3.10 via pyenv
+pyenv install 3.10
+pyenv global 3.10
 
 # alias
 rm /usr/local/bin/pip | true

frontend/.dockerignore

@@ -1,2 +1,33 @@
 # Ignore the .npmrc file
-.npmrc
+.npmrc
+
+# Ignore node_modules
+node_modules
+
+# local env files
+.env.local
+.env.*.local
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Ignore stats.html
+stats.html
+
+# Ignore .turbo
+.turbo
+
+# Ignore tmp directory
+tmp/