Run functional tests script (#44)

* Add functional test scripts Signed-off-by: Eduardo Apolinario <[email protected]> * Add jq Signed-off-by: Eduardo Apolinario <[email protected]> * Use new flytekit Signed-off-by: Eduardo Apolinario <[email protected]> * Temporarily point to locahost Signed-off-by: Eduardo Apolinario <[email protected]> * Hardcode flytesnacks version Signed-off-by: Eduardo Apolinario <[email protected]> * Revert "Temporarily point to locahost" This reverts commit 24e0202f16a509838a3332dc37e8228a92700f83. Signed-off-by: Eduardo Apolinario <[email protected]> * Hardcode flytesnacks version Signed-off-by: Eduardo Apolinario <[email protected]> * Do not register in run.sh Signed-off-by: Eduardo Apolinario <[email protected]> * Run other tests Signed-off-by: Eduardo Apolinario <[email protected]> * Use flytesnacks v0.1.1 Signed-off-by: Eduardo Apolinario <[email protected]> * Remove hardcoded version of flytesnacks Signed-off-by: Eduardo Apolinario <[email protected]> * Register flytesnacks examples from inside the pod Signed-off-by: Eduardo Apolinario <[email protected]> * Use flytekit==0.30.0 Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Feb 4, 2022 · e4b7017 · e4b7017
1 parent 841239f
commit e4b7017
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 6 deletions.
diff --git a/flytetools/flytetester/Dockerfile b/flytetools/flytetester/Dockerfile
@@ -13,7 +13,7 @@ RUN : \
 
 RUN : \
     && apt-get update \
-    && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim
+    && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim jq
 
 # This is necessary for opencv to work
 RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg

diff --git a/flytetools/flytetester/Makefile b/flytetools/flytetester/Makefile
@@ -46,6 +46,10 @@ register_sandbox: docker_push
 end2end_test:
 	flytekit_venv end2end/run.sh
 
+.PHONY: functional_test
+functional_test:
+	flytekit_venv functional-tests/run.sh
+
 .PHONY: docker_build
 docker_build:
 	scripts/docker_build.sh
@@ -56,4 +60,4 @@ docker_push:
 
 .PHONY: docker_build_push
 docker_build_push: docker_build docker_push
-	echo "built"
+	echo "built"
diff --git a/flytetools/flytetester/functional-tests/functional-test.config b/flytetools/flytetester/functional-tests/functional-test.config
@@ -0,0 +1,3 @@
+[platform]
+url=flyteadmin:81
+insecure=True
diff --git a/flytetools/functional-tests/run-tests.py → ...flytetester/functional-tests/run-tests.py b/flytetools/functional-tests/run-tests.py → ...flytetester/functional-tests/run-tests.py
@@ -115,11 +115,11 @@ def valid(workflow_group):
     return workflow_group in FLYTESNACKS_WORKFLOW_GROUPS.keys()
 
 
-def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]:
+def run(release_tag: str, priorities: List[str], config_file_path) -> List[Dict[str, str]]:
     remote = FlyteRemote.from_config(
         default_project="flytesnacks",
         default_domain="development",
-        config_file_path=f"./functional-tests/config",
+        config_file_path=config_file_path,
     )
 
     # For a given release tag and priority, this function filters the workflow groups from the flytesnacks manifest file. For
@@ -178,8 +178,17 @@ def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]:
     # tests manifest.
     flytesnacks_release_tag = sys.argv[1]
     priorities = sys.argv[2].split(',')
+    config_file = sys.argv[3]
+    return_non_zero_on_failure = sys.argv[4]
 
-    results = run(flytesnacks_release_tag, priorities)
+    results = run(flytesnacks_release_tag, priorities, config_file)
 
     # Write a json object in its own line describing the result of this run to stdout
     print(f"Result of run:\n{json.dumps(results)}")
+
+    # Return a non-zero exit code if core fails
+    if return_non_zero_on_failure is not None:
+        # find the result
+        for result in results:
+            if result['label'] == return_non_zero_on_failure and result['status'] != 'passing':
+                sys.exit(1)
diff --git a/flytetools/flytetester/functional-tests/run.sh b/flytetools/flytetester/functional-tests/run.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+
+LATEST_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name)
+
+flytectl register examples -p flytesnacks -d development --config /opt/go/config.yaml
+
+flytekit_venv python functional-tests/run-tests.py $LATEST_VERSION P0,P1 functional-tests/functional-test.config core
diff --git a/flytetools/flytetester/requirements.txt b/flytetools/flytetester/requirements.txt
@@ -1,4 +1,4 @@
-flytekit[sidecar,schema]==0.26.0
+flytekit[sidecar,schema]==0.30.0
 statsd
 opencv-python
 k8s-proto>=0.0.2