Initial commit on Unstract

.github/pull_request_template.md

@@ -0,0 +1,35 @@
+## What
+
+...
+
+## Why
+
+...
+
+## How
+
+...
+
+## Relevant Docs
+
+-
+
+## Related Issues or PRs
+
+-
+
+## Dependencies Versions / Env Variables
+
+-
+
+## Notes on Testing
+
+...
+
+## Screenshots
+
+...
+
+## Checklist
+
+I have read and understood the [Contribution Guidelines]().

.github/workflows/ci-container-build.yaml

@@ -0,0 +1,86 @@
+name: Container Image Build Test for PRs
+
+env:
+  VERSION: ci-test # Used for docker tag
+
+on:
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'backend/**'
+      - 'frontend/**'
+      - 'unstract/**'
+      - 'document-service/**'
+      - 'platform-service/**'
+      - 'x2text-service/**'
+      - 'worker/**'
+      - 'docker/dockerfiles/**'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches:
+      - main
+      - development
+    paths:
+      - 'backend/**'
+      - 'frontend/**'
+      - 'unstract/**'
+      - 'document-service/**'
+      - 'platform-service/**'
+      - 'x2text-service/**'
+      - 'worker/**'
+      - 'docker/dockerfiles/**'
+
+jobs:
+  build:
+    if: github.event.pull_request.draft == false
+    runs-on: custom-k8s-runner
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Container Build
+        working-directory: ./docker
+        run: |
+          docker compose -f docker-compose.build.yaml build
+      - name: Container Run
+        working-directory: ./docker
+        run: |
+          cp ../backend/sample.env ../backend/.env
+          cp ../document-service/sample.env ../document-service/.env
+          cp ../platform-service/sample.env ../platform-service/.env
+          cp ../prompt-service/sample.env ../prompt-service/.env
+          cp ../worker/sample.env ../worker/.env
+          cp ../x2text-service/sample.env ../x2text-service/.env
+          cp sample.essentials.env essentials.env
+
+          docker compose -f docker-compose.yaml up -d
+          sleep 10
+          docker compose -f docker-compose.yaml ps -a
+          # Get the names of exited containers
+          custom_format="{{.Name}}\t{{.Image}}\t{{.Service}}"
+          EXITED_CONTAINERS=$(docker compose -f docker-compose.yaml ps -a --filter status=exited --format "$custom_format")
+
+          line_count=$(echo "$EXITED_CONTAINERS" | wc -l)
+
+          if [ "$line_count" -gt 1 ]; then
+            echo "Exited Containers: $EXITED_CONTAINERS"
+
+            SERVICE=$(echo "$EXITED_CONTAINERS" | awk 'NR>0 {print $3}')
+            echo "Exited Services:"
+            echo "$SERVICE"
+            echo "There are exited containers."
+            # Print logs of exited containers
+            IFS=$'\n'
+            for SERVICE in $SERVICE; do
+              docker compose -f docker-compose.yaml logs "$SERVICE"
+            done
+            docker compose -f docker-compose.yaml down -v
+            exit 1
+          fi
+          docker compose -f docker-compose.yaml down -v

.github/workflows/docker-build-push-dev.yaml

@@ -0,0 +1,64 @@
+name: Unstract Docker Image Build and Push (Development)
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag"
+        required: true
+        default: "latest"
+      service_name:
+        description: "Service to build"
+        required: true
+        default: "backend" # Provide a default value
+        type: choice
+        options: # Define available options
+          - all-services
+          - frontend
+          - backend
+          - document-service
+          - platform-service
+          - worker
+          - prompt-service
+          - x2text-service
+
+run-name: "[${{ inputs.service_name }}] Docker Image Build and Push (Development)"
+
+jobs:
+  build-and-push:
+    runs-on: custom-k8s-runner
+    steps:
+      - name: Output Inputs
+        run: echo "${{ toJSON(github.event.inputs) }}"
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Build and push Docker image for the specified service
+      - name: Build and push image for ${{ github.event.inputs.service_name }}
+        working-directory: ./docker
+        if: github.event.inputs.service_name != 'all-services'
+        run: |
+          VERSION=${{ github.event.inputs.tag }} docker compose -f docker-compose.build.yaml build --no-cache ${{ github.event.inputs.service_name }}
+          docker push unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}
+
+      # Build and push all service images
+      - name: Build and push all images
+        working-directory: ./docker
+        if: github.event.inputs.service_name == 'all-services'
+        run: |
+          VERSION=${{ github.event.inputs.tag }} docker-compose -f docker-compose.build.yaml build --no-cache
+          # Push all built images
+          docker push unstract/backend:${{ github.event.inputs.tag }}
+          docker push unstract/frontend:${{ github.event.inputs.tag }}
+          docker push unstract/document-service:${{ github.event.inputs.tag }}
+          docker push unstract/platform-service:${{ github.event.inputs.tag }}
+          docker push unstract/worker:${{ github.event.inputs.tag }}
+          docker push unstract/prompt-service:${{ github.event.inputs.tag }}
+          docker push unstract/x2text-service:${{ github.event.inputs.tag }}

.github/workflows/docker-tools-build-push.yaml

@@ -0,0 +1,65 @@
+name: Unstract Tools Docker Image Build and Push (Development)
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag"
+        required: true
+        default: "latest"
+      service_name:
+        description: "Tool to build"
+        required: true
+        default: "tool-classifier" # Provide a default value
+        type: choice
+        options: # Define available options
+          - tool-classifier
+          - tool-doc-pii-redactor
+          - tool-indexer
+          - tool-ocr
+          - tool-translate
+          - tool-structure
+          - tool-text-extractor
+
+run-name: "[${{ inputs.service_name }}] Docker Image Build and Push (Development)"
+
+jobs:
+    build-and-push:
+      runs-on: custom-k8s-runner
+      steps:
+        - name: Output Inputs
+          run: echo "${{ toJSON(github.event.inputs) }}"
+
+        - name: Checkout code
+          uses: actions/checkout@v2
+
+        - name: Login to Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        - name: Build tool-classifier
+          if: github.event.inputs.service_name=='tool-classifier'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/classifier
+        - name: Build tool-doc-pii-redactor
+          if: github.event.inputs.service_name=='tool-doc-pii-redactor'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/doc_pii_redactor
+        - name: Build tool-indexer
+          if: github.event.inputs.service_name=='tool-indexer'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/indexer
+        - name: Build tool-ocr
+          if: github.event.inputs.service_name=='tool-ocr'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/ocr
+        - name: Build tool-translate
+          if: github.event.inputs.service_name=='tool-translate'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/translate
+        - name: Build tool-structure
+          if: github.event.inputs.service_name=='tool-structure'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/structure
+        - name: Build tool-text-extractor
+          if: github.event.inputs.service_name=='tool-text-extractor'
+          run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/text_extractor
+
+        - name: Push Docker image to Docker Hub
+          run: docker push unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}

.github/workflows/production-build.yaml

@@ -0,0 +1,33 @@
+name: Unstract Docker Image Build and Push (Production)
+
+on:
+  release:
+    types:
+      - created
+
+run-name: "[${{ github.event.release.tag_name }}] Docker Image Build and Push (Development)"
+
+jobs:
+  build-and-push:
+    runs-on: custom-k8s-runner
+    strategy:
+      matrix:          
+        service_name: [backend, frontend, document-service, platform-service, prompt-service, worker, x2text-service]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.release.tag_name }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        working-directory: ./docker
+        run: |
+          VERSION=${{ github.event.release.tag_name }} docker-compose -f docker-compose.build.yaml build --no-cache ${{ matrix.service_name }}
+          docker push unstract/${{ matrix.service_name }}:${{ github.event.release.tag_name }}