Re-add cache service (#111)

* re-add cache service

Signed-off-by: Paul Dittamo <[email protected]>

* add redis output and reservation clients

Signed-off-by: Paul Dittamo <[email protected]>

* lint

Signed-off-by: Paul Dittamo <[email protected]>

* default to use already existing output path for caching

Signed-off-by: Paul Dittamo <[email protected]>

* add error handling redis client start up

Signed-off-by: Paul Dittamo <[email protected]>

* set default caching data stores to be in memory

Signed-off-by: Paul Dittamo <[email protected]>

* improve log levels

Signed-off-by: Paul Dittamo <[email protected]>

* typo

Signed-off-by: Paul Dittamo <[email protected]>

* only allow RemoteFileOutputReader for offloaded caching

Signed-off-by: Paul Dittamo <[email protected]>

* dummy commit

Signed-off-by: Paul Dittamo <[email protected]>

* clean up log levels in cache client

Signed-off-by: Paul Dittamo <[email protected]>

* add cache service to helm charts

Signed-off-by: Paul Dittamo <[email protected]>

* add cacheservice to various release scripts

Signed-off-by: Paul Dittamo <[email protected]>

* Revert "add cacheservice to various release scripts"

This reverts commit 66d3a05c0febb5023de2ae2e79974436334c5cb0.

Signed-off-by: Paul Dittamo <[email protected]>

* Revert "add cache service to helm charts"

This reverts commit 66af6de38f739978f2656543ea85106d71790c17.

---------

Signed-off-by: Paul Dittamo <[email protected]>

.github/workflows/build-and-push-all-docker-images.yml

@@ -14,6 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         component:
+          - cacheservice
           - datacatalog
           - flyteadmin
           - flytecopilot

.github/workflows/checks.yml

@@ -30,6 +30,7 @@ jobs:
       fail-fast: false
       matrix:
         component:
+          - cacheservice
           - datacatalog
           - flyteadmin
           # TODO(monorepo): Enable lint flytecopilot
@@ -50,6 +51,7 @@ jobs:
       fail-fast: false
       matrix:
         component:
+          - cacheservice
           - datacatalog
           - flyteadmin
           - flytecopilot
@@ -65,6 +67,7 @@ jobs:
       fail-fast: false
       matrix:
         component:
+          - cacheservice
           - datacatalog
           - flyteadmin
           - flytecopilot
@@ -108,6 +111,7 @@ jobs:
       fail-fast: false
       matrix:
         component:
+          - cacheservice
           - datacatalog
           - flyteadmin
           - flytecopilot

.github/workflows/end2end.yml

@@ -30,6 +30,11 @@ jobs:
       - uses: actions/setup-python@v3
         with:
           python-version: 3.11
+      - id: load-docker-cache-cacheservice
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-cacheservice
+          key: ${{ inputs.cache_key }}-cacheservice
       - id: load-docker-cache-datacatalog
         uses: actions/cache@v3
         with:
@@ -52,6 +57,7 @@ jobs:
           key: ${{ inputs.cache_key }}-flytepropeller
       - name: Create Sandbox Cluster
         run: |
+          cp /tmp/tmp/docker-images-cacheservice/snapshot-cacheservice.tar snapshot-cacheservice.tar
           cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar
           cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar
           cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar
@@ -60,6 +66,7 @@ jobs:
           flytectl sandbox start --source=$(pwd)
       - name: Prime docker cache
         run: |
+          flytectl sandbox exec -- docker load -i /root/snapshot-cacheservice.tar
           flytectl sandbox exec -- docker load -i /root/snapshot-datacatalog.tar
           flytectl sandbox exec -- docker load -i /root/snapshot-flyteadmin.tar
           flytectl sandbox exec -- docker load -i /root/snapshot-flytecopilot.tar
@@ -107,10 +114,11 @@ jobs:
         run: |
           flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
           flytectl sandbox exec -- helm repo update
-          flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
+          flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest,cacheservice.image.repository=${{ github.repository_owner }}/cacheservice,cacheservice.image.tag=latest
           # TODO(monorepo): the following commands are not correct.
           # we have to separate the calls to replace the images into different commands.
           # flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \
+          #   --set cacheservice.image.repository=${{ github.repository_owner }}/cacheservice,cacheservice.image.tag=latest
           #   --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
           #   --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest
           #   --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest

Dockerfile

@@ -10,6 +10,7 @@ ENV GOOS linux
 
 WORKDIR /flyteorg/build
 
+COPY cacheservice cacheservice
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
 COPY flytecopilot flytecopilot

Dockerfile.cacheservice

@@ -0,0 +1,53 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+
+ARG TARGETARCH
+ENV GOARCH "${TARGETARCH}"
+ENV GOOS linux
+
+RUN apk add git openssh-client make curl
+
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+# Pull GRPC health probe binary for liveness and readiness checks
+RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \
+    wget -qO/artifacts/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /artifacts/grpc_health_probe && \
+    echo 'ded15e598d887ccc47bf2321371950bbf930f5e4856b9f75712ce4b2b5120480  /artifacts/grpc_health_probe' > .grpc_checksum && \
+    sha256sum -c .grpc_checksum
+
+WORKDIR /go/src/github.com/flyteorg/cacheservice
+
+COPY cacheservice .
+COPY flyteadmin ../flyteadmin
+COPY flytecopilot ../flytecopilot
+COPY flyteidl ../flyteidl
+COPY flyteplugins ../flyteplugins
+COPY flytepropeller ../flytepropeller
+COPY flytestdlib ../flytestdlib
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/cacheservice
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.18
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/cacheservice
+
+COPY --from=builder /artifacts /bin
+
+# Ensure the latest CA certs are present to authenticate SSL connections.
+RUN apk --update add ca-certificates
+
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
+CMD ["cacheservice"]

Dockerfile.flyteadmin

@@ -23,6 +23,7 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \
 
 WORKDIR /go/src/github.com/flyteorg/flyteadmin
 
+COPY cacheservice ../cacheservice
 COPY datacatalog ../datacatalog
 COPY flyteadmin .
 COPY flytecopilot ../flytecopilot

Dockerfile.flytecopilot

@@ -13,6 +13,7 @@ RUN apk add git openssh-client make curl
 
 WORKDIR /go/src/github.com/flyteorg/flytecopilot
 
+COPY cacheservice ../cacheservice
 COPY datacatalog ../datacatalog
 COPY flyteadmin ../flyteadmin
 COPY flytecopilot .

Dockerfile.flytepropeller

@@ -13,6 +13,7 @@ RUN apk add git openssh-client make curl
 
 WORKDIR /go/src/github.com/flyteorg/flytepropeller
 
+COPY cacheservice ../cacheservice
 COPY datacatalog ../datacatalog
 COPY flyteadmin ../flyteadmin
 COPY flytecopilot ../flytecopilot

Dockerfile.flytescheduler

@@ -16,6 +16,7 @@ RUN mkdir /artifacts
 
 WORKDIR /go/src/github.com/flyteorg/flyteadmin
 
+COPY cacheservice ../cacheservice
 COPY datacatalog ../datacatalog
 COPY flyteadmin .
 COPY flytecopilot ../flytecopilot

Dockerfile.sandbox-lite

@@ -15,6 +15,7 @@ RUN apk add --no-cache build-base
 COPY go.mod go.sum /app/flyte/
 WORKDIR /app/flyte
 
+COPY cacheservice cacheservice
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
 COPY flytecopilot flytecopilot

Makefile

@@ -108,6 +108,7 @@ build_native_flyte:
 .PHONY: go-tidy
 go-tidy:
 	go mod tidy
+	make -C cacheservice go-tidy
 	make -C datacatalog go-tidy
 	make -C flyteadmin go-tidy
 	make -C flyteidl go-tidy

cacheservice/.gitignore

@@ -0,0 +1,5 @@
+.idea/
+vendor/
+vendor-new/
+.DS_Store
+bin/

cacheservice/.golangci.yml

@@ -0,0 +1,40 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+run:
+  skip-dirs:
+    - pkg/client
+
+linters:
+  disable-all: true
+  enable:
+    - deadcode
+    - errcheck
+    - gas
+    - gci
+    - goconst
+    - goimports
+    - golint
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+
+linters-settings:
+  gci:
+    custom-order: true
+    sections:
+      - standard
+      - default
+      - prefix(github.com/flyteorg)
+    skip-generated: true

cacheservice/Makefile

@@ -0,0 +1,28 @@
+export REPOSITORY=cacheservice
+include boilerplate/flyte/docker_build/Makefile
+include boilerplate/flyte/golang_test_targets/Makefile
+
+.PHONY: update_boilerplate
+update_boilerplate:
+	@curl https://raw.githubusercontent.com/flyteorg/boilerplate/master/boilerplate/update.sh -o boilerplate/update.sh
+	@boilerplate/update.sh
+
+.PHONY: compile
+compile:
+	mkdir -p ./bin
+	go build -o bin/cacheservice ./cmd/main.go
+
+.PHONY: linux_compile
+linux_compile: export CGO_ENABLED ?= 0
+linux_compile: export GOOS ?= linux
+linux_compile:
+	go build -o /artifacts/cacheservice ./cmd/
+
+.PHONY: generate
+generate:
+	@go generate ./...
+
+
+.PHONY: go-tidy
+go-tidy:
+	go mod tidy

cacheservice/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md

@@ -0,0 +1,2 @@
+This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). 
+All contributors and participants agree to abide by its terms.

cacheservice/boilerplate/flyte/code_of_conduct/README.rst

@@ -0,0 +1,2 @@
+CODE OF CONDUCT
+~~~~~~~~~~~~~~~

cacheservice/boilerplate/flyte/code_of_conduct/update.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+cp ${DIR}/CODE_OF_CONDUCT.md ${DIR}/../../../CODE_OF_CONDUCT.md

cacheservice/boilerplate/flyte/docker_build/Makefile

@@ -0,0 +1,12 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+.PHONY: docker_build
+docker_build:
+	IMAGE_NAME=$$REPOSITORY ./boilerplate/flyte/docker_build/docker_build.sh
+
+.PHONY: dockerhub_push
+dockerhub_push:
+	IMAGE_NAME=flyteorg/$$REPOSITORY REGISTRY=docker.io ./boilerplate/flyte/docker_build/docker_build.sh

cacheservice/boilerplate/flyte/docker_build/Readme.rst

@@ -0,0 +1,23 @@
+Docker Build and Push
+~~~~~~~~~~~~~~~~~~~~~
+
+Provides a ``make docker_build`` target that builds your image locally.
+
+Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub.
+
+The Dockerhub image will tagged ``<REPOSITORY>:<GIT COMMIT SHA>``
+
+If git head has a git tag, the Dockerhub image will also be tagged ``<IMAGE>:<GIT_TAG>``.
+
+**To Enable:**
+
+Add ``flyteorg/docker_build`` to your ``boilerplate/update.cfg`` file.
+
+Add ``include boilerplate/flyte/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+    REPOSITORY=<myreponame>
+    include boilerplate/flyte/docker_build/Makefile
+
+(this ensures the extra Make targets get included in your main Makefile)

cacheservice/boilerplate/flyte/docker_build/docker_build.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+echo ""
+echo "------------------------------------"
+echo "           DOCKER BUILD"
+echo "------------------------------------"
+echo ""
+
+if [ -n "$REGISTRY" ]; then
+  # Do not push if there are unstaged git changes
+  CHANGED=$(git status --porcelain)
+  if [ -n "$CHANGED" ]; then
+    echo "Please commit git changes before pushing to a registry"
+    exit 1
+  fi
+fi
+
+
+GIT_SHA=$(git rev-parse HEAD)
+
+IMAGE_TAG_SUFFIX=""
+# for intermediate build phases, append -$BUILD_PHASE to all image tags
+if [ -n "$BUILD_PHASE" ]; then
+  IMAGE_TAG_SUFFIX="-${BUILD_PHASE}"
+fi
+
+IMAGE_TAG_WITH_SHA="${IMAGE_NAME}:${GIT_SHA}${IMAGE_TAG_SUFFIX}"
+
+RELEASE_SEMVER=$(git describe --tags --exact-match "$GIT_SHA" 2>/dev/null) || true
+if [ -n "$RELEASE_SEMVER" ]; then
+  IMAGE_TAG_WITH_SEMVER="${IMAGE_NAME}:${RELEASE_SEMVER}${IMAGE_TAG_SUFFIX}"
+fi
+
+# build the image
+# passing no build phase will build the final image
+docker build -t "$IMAGE_TAG_WITH_SHA" --target=${BUILD_PHASE} .
+echo "${IMAGE_TAG_WITH_SHA} built locally."
+
+# if REGISTRY specified, push the images to the remote registry
+if [ -n "$REGISTRY" ]; then
+
+  if [ -n "${DOCKER_REGISTRY_PASSWORD}" ]; then
+    docker login --username="$DOCKER_REGISTRY_USERNAME" --password="$DOCKER_REGISTRY_PASSWORD"
+  fi
+
+  docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+
+  docker push "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+  echo "${REGISTRY}/${IMAGE_TAG_WITH_SHA} pushed to remote."
+
+  # If the current commit has a semver tag, also push the images with the semver tag
+  if [ -n "$RELEASE_SEMVER" ]; then
+
+    docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+
+    docker push "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+    echo "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER} pushed to remote."
+
+  fi
+fi