flytectl sandbox for ci (flyteorg#292)

* flytectl for ci

Signed-off-by: Yuvraj <[email protected]>

.github/workflows/ghcr_push.yml

@@ -58,4 +58,4 @@ jobs:
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ matrix.directory.name }}-${{ github.sha }}"
           context: ./cookbook/${{ matrix.directory.path }}
-          dockerfile: ${{ matrix.directory.name }}/Dockerfile
+          dockerfile: ${{ matrix.directory.name }}/Dockerfile

.github/workflows/master.yml

@@ -22,64 +22,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
           WITH_V: true
-          DEFAULT_BUMP: patch
-  examples-releaser:
-    name: Release Examples
-    strategy:
-      matrix:
-        directory: [ "core", "integrations", "case_studies" ]
-    runs-on: ubuntu-latest
-    needs: [ bump-version ]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - uses: actions/cache@v1
-        with:
-          path: /tmp/release-snacks-${{ matrix.directory }}/
-          key: release-snacks-${{ matrix.directory }}
-      - name: Generate Serialize proto
-        env:
-          REGISTRY: "ghcr.io/flyteorg"
-        run: |
-          make -C cookbook/${{ matrix.directory }} serialize
-          mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/
-          make -C cookbook archive
-          cp -r cookbook/release-snacks/* /tmp/release-snacks-${{ matrix.directory }}/
-  goreleaser:
-    name: Release Case studies example
-    runs-on: ubuntu-latest
-    needs: [ examples-releaser ]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - uses: actions/cache@v1
-        with:
-          path: /tmp/release-snacks-core/
-          key: release-snacks-core
-      - uses: actions/cache@v1
-        with:
-          path: /tmp/release-snacks-case-studies/
-          key: release-snacks-case-studies
-      - uses: actions/cache@v1
-        with:
-          path: /tmp/release-snacks-integrations/
-          key: release-snacks-plugins
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-          ref: ${{ steps.bump-version.outputs.tag }}
-      - name: Copy artifacts
-        run: |
-          mkdir -p cookbook/release-snacks
-          cp -r /tmp/release-snacks-core/* cookbook/release-snacks/
-          cp -r /tmp/release-snacks-case-studies/* cookbook/release-snacks/
-          cp -r /tmp/release-snacks-integrations/* cookbook/release-snacks/
-      - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
-        with:
-          version: latest
-          args: release --rm-dist
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
+          DEFAULT_BUMP: patch

.github/workflows/release_examples.yml

@@ -0,0 +1,78 @@
+name: Serialize & Release Examples
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'cookbook/docs/**'
+      - 'cookbook/deployment/**'
+  create:
+    tags:
+      - v*
+
+jobs:
+  serialize-example:
+    name: Serialize
+    strategy:
+      matrix:
+        directory: [ "core", "integrations", "case_studies" ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: actions/cache@v1
+        if: github.event_name != 'pull_request'
+        with:
+          path: /tmp/release-snacks-${{ matrix.directory }}/
+          key: release-snacks-${{ matrix.directory }}-${{ github.sha }}
+      - name: Serialize Examples
+        env:
+          REGISTRY: "ghcr.io/flyteorg"
+        run: |
+          REGISTRY=ghcr.io/flyteorg VERSION=latest make -C cookbook/${{ matrix.directory }} serialize
+      - name: Cache Serialized protos
+        if: github.event_name != 'pull_request'
+        env:
+          REGISTRY: "ghcr.io/flyteorg"
+        run: |
+          mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/
+          make -C cookbook archive
+          cp -r /tmp/release-snacks-${{ matrix.directory }}/* cookbook/release-snacks/
+  releaser:
+    name: Release example
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: [ serialize-example ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: actions/cache@v1
+        with:
+          path: /tmp/release-snacks-core/
+          key: release-snacks-core-${{ github.sha }}
+      - uses: actions/cache@v1
+        with:
+          path: /tmp/release-snacks-case_studies/
+          key: release-snacks-case_studies-${{ github.sha }}
+      - uses: actions/cache@v1
+        with:
+          path: /tmp/release-snacks-integrations/
+          key: release-snacks-integrations-${{ github.sha }}
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+          ref: ${{ steps.bump-version.outputs.tag }}
+      - name: Copy artifacts
+        run: |
+          mkdir -p cookbook/release-snacks
+          cp -r /tmp/release-snacks-core/* cookbook/release-snacks/
+          cp -r /tmp/release-snacks-case_studies/* cookbook/release-snacks/
+          cp -r /tmp/release-snacks-integrations/* cookbook/release-snacks/
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}

.github/workflows/sandbox_register.yml

@@ -21,17 +21,8 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: "0"
-      - run: docker pull ghcr.io/flyteorg/flyte-sandbox:dind
+      - uses: unionai/[email protected]
       - name: setup sandbox
-        run: make setup
-      - name: wait for sandbox
-        run: |
-          n=0
-          until [ "$n" -ge 5 ]; do
-            echo "try $n"
-            make wait && break
-            n=$((n+1))
-            sleep 1
-          done
+        run: flytectl sandbox start --sourcesPath=$GITHUB_WORKSPACE
       - name: fast register
-        run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register
+        run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register

Makefile

@@ -8,7 +8,7 @@ FLYTE_PROXY_PORT := 30081
 K8S_DASHBOARD_PROXY_PORT := 30082
 MINIO_PROXY_PORT := 30084
 FLYTE_SANDBOX_NAME := flyte-sandbox
-KUBE_CONFIG := ~/kubeconfig
+FLYTE_DIR := ~/.flyte
 
 # Module of cookbook examples to register
 EXAMPLES_MODULE := core
@@ -33,6 +33,7 @@ update_boilerplate:
 	@curl https://raw.githubusercontent.com/flyteorg/boilerplate/master/boilerplate/update.sh -o boilerplate/update.sh
 	@boilerplate/update.sh
 
+
 .PHONY: help
 help: ## Show help message
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -47,25 +48,10 @@ endif
 .PHONY: setup
 setup:
 	$(call LOG,Starting Flyte sandbox)
-	docker run -d --rm --privileged --name $(FLYTE_SANDBOX_NAME) \
-		-e SANDBOX=1 \
-		-e KUBERNETES_API_PORT=$(KUBERNETES_API_PORT) \
-		-e FLYTE_HOST=localhost:30081 \
-		-e FLYTE_AWS_ENDPOINT=http://localhost:30084/ \
-		-v $(CURDIR):/usr/src \
-		-v $(KUBE_CONFIG):/etc/rancher/ \
-		-p $(KUBERNETES_API_PORT):$(KUBERNETES_API_PORT) \
-		-p $(FLYTE_PROXY_PORT):30081 \
-		-p $(K8S_DASHBOARD_PROXY_PORT):30082 \
-		-p $(MINIO_PROXY_PORT):30084 \
-		cr.flyte.org/flyteorg/flyte-sandbox:dind > /dev/null
-
-.PHONY: wait
-wait:
-	$(call RUN_IN_SANDBOX, wait-for-flyte.sh)
+	flytectl sandbox start --sourcesPath:=$(shell pwd)
 
 .PHONY: start
-start: setup wait ## Start a local Flyte sandbox
+start: setup
 	$(call LOG,Registering examples from commit: latest)
 	REGISTRY=cr.flyte.org/flyteorg VERSION=latest $(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_register)
 
@@ -74,7 +60,7 @@ start: setup wait ## Start a local Flyte sandbox
 .PHONY: teardown
 teardown: _requires-sandbox-up  ## Teardown Flyte sandbox
 	$(call LOG,Tearing down Flyte sandbox)
-	docker rm -f -v $(FLYTE_SANDBOX_NAME) > /dev/null ||:
+	flytectl sandbox teardown
 
 .PHONY: status
 status: _requires-sandbox-up  ## Show status of Flyte deployment
@@ -95,7 +81,7 @@ fast_register: _requires-sandbox-up  ## Fast register Flyte cookbook workflows
 	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_register)
 
 .PHONY: setup-kubectl
-kubectl-config: 
+kubectl-config:
 	# In shell/bash, run: `eval $(make kubectl-config)`
 	# Makefiles run recipes in sub-processes. A sub-process cannot modify the parent process's environment.
 	# The best I (@EngHabu) can think of at the moment is to output this for the user to eval in the

cookbook/Makefile

@@ -31,4 +31,8 @@ run-examples: ## Runs all examples
 
 .PHONY: archive
 archive: # archive directories (e.g. _pb_output/)
-	sh scripts/archive-all-example.sh
+	bash scripts/ci_helper.sh RELEASE
+
+.PHONY: register-workflow-ci
+register-workflow-ci:
+	bash scripts/ci_helper.sh REGISTER

cookbook/flyte_tests_manifest.json

@@ -81,7 +81,7 @@
  },{
    "name": "case-studies-pima-diabetes",
    "priority": "P2",
-   "path": "case_studies/ml_training/pima-diabetes",
+   "path": "case_studies/ml_training/pima_diabetes",
    "exitCondition": {
      "exit_success": true,
      "exit_message": ""

cookbook/scripts/ci_helper.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+release_example() {
+  for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do
+    if [ -d "./$(echo ${row} | jq -r '.path')/_pb_output/" ]; then
+      tar -cvzf "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz"  "./$(echo ${row} | jq -r '.path')/_pb_output/"
+    fi
+  done
+}
+
+register_example() {
+  for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do
+    flytectl register file "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" -p flytesnacks -d development --archive
+  done
+}
+
+if [ "$1" == "RELEASE" ]; then
+  release_example
+elif [ $1 == "REGISTER" ]; then
+  register_example
+fi