From e4b70175be841b66e31a3ea7239518ef5ccf0ed8 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Thu, 3 Feb 2022 19:47:26 -0800 Subject: [PATCH] Run functional tests script (#44) * Add functional test scripts Signed-off-by: Eduardo Apolinario * Add jq Signed-off-by: Eduardo Apolinario * Use new flytekit Signed-off-by: Eduardo Apolinario * Temporarily point to locahost Signed-off-by: Eduardo Apolinario * Hardcode flytesnacks version Signed-off-by: Eduardo Apolinario * Revert "Temporarily point to locahost" This reverts commit 24e0202f16a509838a3332dc37e8228a92700f83. Signed-off-by: Eduardo Apolinario * Hardcode flytesnacks version Signed-off-by: Eduardo Apolinario * Do not register in run.sh Signed-off-by: Eduardo Apolinario * Run other tests Signed-off-by: Eduardo Apolinario * Use flytesnacks v0.1.1 Signed-off-by: Eduardo Apolinario * Remove hardcoded version of flytesnacks Signed-off-by: Eduardo Apolinario * Register flytesnacks examples from inside the pod Signed-off-by: Eduardo Apolinario * Use flytekit==0.30.0 Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- flytetools/flytetester/Dockerfile | 2 +- flytetools/flytetester/Makefile | 6 +++++- .../functional-tests/functional-test.config | 3 +++ .../functional-tests/run-tests.py | 15 ++++++++++++--- flytetools/flytetester/functional-tests/run.sh | 9 +++++++++ flytetools/flytetester/requirements.txt | 2 +- 6 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 flytetools/flytetester/functional-tests/functional-test.config rename flytetools/{ => flytetester}/functional-tests/run-tests.py (93%) create mode 100755 flytetools/flytetester/functional-tests/run.sh diff --git a/flytetools/flytetester/Dockerfile b/flytetools/flytetester/Dockerfile index 97ac031a50f..f4b4cf06dc8 100644 --- a/flytetools/flytetester/Dockerfile +++ b/flytetools/flytetester/Dockerfile @@ -13,7 +13,7 @@ RUN : \ RUN : \ && apt-get update \ - && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim + && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim jq # This is necessary for opencv to work RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg diff --git a/flytetools/flytetester/Makefile b/flytetools/flytetester/Makefile index e5a57604df8..2b28059a0df 100644 --- a/flytetools/flytetester/Makefile +++ b/flytetools/flytetester/Makefile @@ -46,6 +46,10 @@ register_sandbox: docker_push end2end_test: flytekit_venv end2end/run.sh +.PHONY: functional_test +functional_test: + flytekit_venv functional-tests/run.sh + .PHONY: docker_build docker_build: scripts/docker_build.sh @@ -56,4 +60,4 @@ docker_push: .PHONY: docker_build_push docker_build_push: docker_build docker_push - echo "built" \ No newline at end of file + echo "built" diff --git a/flytetools/flytetester/functional-tests/functional-test.config b/flytetools/flytetester/functional-tests/functional-test.config new file mode 100644 index 00000000000..2520284f386 --- /dev/null +++ b/flytetools/flytetester/functional-tests/functional-test.config @@ -0,0 +1,3 @@ +[platform] +url=flyteadmin:81 +insecure=True diff --git a/flytetools/functional-tests/run-tests.py b/flytetools/flytetester/functional-tests/run-tests.py similarity index 93% rename from flytetools/functional-tests/run-tests.py rename to flytetools/flytetester/functional-tests/run-tests.py index 8e51339d216..3f987969d99 100644 --- a/flytetools/functional-tests/run-tests.py +++ b/flytetools/flytetester/functional-tests/run-tests.py @@ -115,11 +115,11 @@ def valid(workflow_group): return workflow_group in FLYTESNACKS_WORKFLOW_GROUPS.keys() -def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]: +def run(release_tag: str, priorities: List[str], config_file_path) -> List[Dict[str, str]]: remote = FlyteRemote.from_config( default_project="flytesnacks", default_domain="development", - config_file_path=f"./functional-tests/config", + config_file_path=config_file_path, ) # For a given release tag and priority, this function filters the workflow groups from the flytesnacks manifest file. For @@ -178,8 +178,17 @@ def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]: # tests manifest. flytesnacks_release_tag = sys.argv[1] priorities = sys.argv[2].split(',') + config_file = sys.argv[3] + return_non_zero_on_failure = sys.argv[4] - results = run(flytesnacks_release_tag, priorities) + results = run(flytesnacks_release_tag, priorities, config_file) # Write a json object in its own line describing the result of this run to stdout print(f"Result of run:\n{json.dumps(results)}") + + # Return a non-zero exit code if core fails + if return_non_zero_on_failure is not None: + # find the result + for result in results: + if result['label'] == return_non_zero_on_failure and result['status'] != 'passing': + sys.exit(1) diff --git a/flytetools/flytetester/functional-tests/run.sh b/flytetools/flytetester/functional-tests/run.sh new file mode 100755 index 00000000000..0d341022933 --- /dev/null +++ b/flytetools/flytetester/functional-tests/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +LATEST_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name) + +flytectl register examples -p flytesnacks -d development --config /opt/go/config.yaml + +flytekit_venv python functional-tests/run-tests.py $LATEST_VERSION P0,P1 functional-tests/functional-test.config core diff --git a/flytetools/flytetester/requirements.txt b/flytetools/flytetester/requirements.txt index 5c0907d5f06..c29c4d1c1ca 100644 --- a/flytetools/flytetester/requirements.txt +++ b/flytetools/flytetester/requirements.txt @@ -1,4 +1,4 @@ -flytekit[sidecar,schema]==0.26.0 +flytekit[sidecar,schema]==0.30.0 statsd opencv-python k8s-proto>=0.0.2