diff --git a/flytetools/flytetester/Dockerfile b/flytetools/flytetester/Dockerfile index 97ac031a50f..f4b4cf06dc8 100644 --- a/flytetools/flytetester/Dockerfile +++ b/flytetools/flytetester/Dockerfile @@ -13,7 +13,7 @@ RUN : \ RUN : \ && apt-get update \ - && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim + && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim jq # This is necessary for opencv to work RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg diff --git a/flytetools/flytetester/Makefile b/flytetools/flytetester/Makefile index e5a57604df8..2b28059a0df 100644 --- a/flytetools/flytetester/Makefile +++ b/flytetools/flytetester/Makefile @@ -46,6 +46,10 @@ register_sandbox: docker_push end2end_test: flytekit_venv end2end/run.sh +.PHONY: functional_test +functional_test: + flytekit_venv functional-tests/run.sh + .PHONY: docker_build docker_build: scripts/docker_build.sh @@ -56,4 +60,4 @@ docker_push: .PHONY: docker_build_push docker_build_push: docker_build docker_push - echo "built" \ No newline at end of file + echo "built" diff --git a/flytetools/flytetester/functional-tests/functional-test.config b/flytetools/flytetester/functional-tests/functional-test.config new file mode 100644 index 00000000000..2520284f386 --- /dev/null +++ b/flytetools/flytetester/functional-tests/functional-test.config @@ -0,0 +1,3 @@ +[platform] +url=flyteadmin:81 +insecure=True diff --git a/flytetools/functional-tests/run-tests.py b/flytetools/flytetester/functional-tests/run-tests.py similarity index 93% rename from flytetools/functional-tests/run-tests.py rename to flytetools/flytetester/functional-tests/run-tests.py index 8e51339d216..3f987969d99 100644 --- a/flytetools/functional-tests/run-tests.py +++ b/flytetools/flytetester/functional-tests/run-tests.py @@ -115,11 +115,11 @@ def valid(workflow_group): return workflow_group in FLYTESNACKS_WORKFLOW_GROUPS.keys() -def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]: +def run(release_tag: str, priorities: List[str], config_file_path) -> List[Dict[str, str]]: remote = FlyteRemote.from_config( default_project="flytesnacks", default_domain="development", - config_file_path=f"./functional-tests/config", + config_file_path=config_file_path, ) # For a given release tag and priority, this function filters the workflow groups from the flytesnacks manifest file. For @@ -178,8 +178,17 @@ def run(release_tag: str, priorities: List[str]) -> List[Dict[str, str]]: # tests manifest. flytesnacks_release_tag = sys.argv[1] priorities = sys.argv[2].split(',') + config_file = sys.argv[3] + return_non_zero_on_failure = sys.argv[4] - results = run(flytesnacks_release_tag, priorities) + results = run(flytesnacks_release_tag, priorities, config_file) # Write a json object in its own line describing the result of this run to stdout print(f"Result of run:\n{json.dumps(results)}") + + # Return a non-zero exit code if core fails + if return_non_zero_on_failure is not None: + # find the result + for result in results: + if result['label'] == return_non_zero_on_failure and result['status'] != 'passing': + sys.exit(1) diff --git a/flytetools/flytetester/functional-tests/run.sh b/flytetools/flytetester/functional-tests/run.sh new file mode 100755 index 00000000000..0d341022933 --- /dev/null +++ b/flytetools/flytetester/functional-tests/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +LATEST_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name) + +flytectl register examples -p flytesnacks -d development --config /opt/go/config.yaml + +flytekit_venv python functional-tests/run-tests.py $LATEST_VERSION P0,P1 functional-tests/functional-test.config core diff --git a/flytetools/flytetester/requirements.txt b/flytetools/flytetester/requirements.txt index 5c0907d5f06..c29c4d1c1ca 100644 --- a/flytetools/flytetester/requirements.txt +++ b/flytetools/flytetester/requirements.txt @@ -1,4 +1,4 @@ -flytekit[sidecar,schema]==0.26.0 +flytekit[sidecar,schema]==0.30.0 statsd opencv-python k8s-proto>=0.0.2