Kubernetes Cluster is started on host not in the container (#8265)

Tests requiring Kubernetes Cluster are now moved out of
the regular CI tests and moved to "kubernetes_tests" folder
so that they can be run entirely on host without having
the CI image built at all. They use production image
to run the tests on KinD cluster and we add tooling
to start/stop/deploy the application to the KinD cluster
automatically - for both CI testing and local development.

This is a pre-requisite to convert the tests to convert the
tests to use the official Helm Chart and Docker images or
Apache Airflow.

It closes #8782

.github/workflows/ci.yml

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
 ---
 name: CI Build
 on:
@@ -25,7 +24,9 @@ on:
     branches: ['master', 'v1-10-test', 'v1-10-stable']
   pull_request:
     branches: ['master', 'v1-10-test', 'v1-10-stable']
+
 env:
+
   MOUNT_LOCAL_SOURCES: "false"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   SKIP_CHECK_REMOTE_IMAGE: "true"
@@ -38,7 +39,9 @@ env:
   CACHE_IMAGE_PREFIX: ${{ github.repository }}
   CACHE_REGISTRY_USERNAME: ${{ github.actor }}
   CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
 jobs:
+
   static-checks-1:
     timeout-minutes: 60
     name: "Checks: some checks"
@@ -52,9 +55,9 @@ jobs:
         with:
           python-version: '3.x'
       - name: Cache pre-commit env
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         env:
-          cache-name: cache-pre-commit
+          cache-name: cache-pre-commit-v2
         with:
           path: ~/.cache/pre-commit
           key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
@@ -166,17 +169,18 @@ jobs:
           ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} '\.py$|.github/workflows/|^Dockerfile'
           echo "::set-output name=count::$?"
         id: pyfiles
+
   tests-kubernetes:
     timeout-minutes: 80
-    name: "kubernetes-test-${{matrix.kube-mode}}-kube"
+    name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}} ${{matrix.kubernetes-version}}"
     runs-on: ubuntu-latest
     needs: [static-checks-1, static-checks-2, pyfiles]
     strategy:
       matrix:
+        python-version: [3.6, 3.7]
         kube-mode:
-          - persistent_mode
-#          #TODO These tests will be unblocked when k8s tests run on host
-#          - git_mode
+          - image
+          - git
         kubernetes-version:
           - "v1.15.3"
       fail-fast: false
@@ -185,25 +189,38 @@ jobs:
       TEST_TYPE: ${{ matrix.test-type }}
       RUN_TESTS: "true"
       CI_JOB_TYPE: "Tests"
-      PYTHON_MAJOR_MINOR_VERSION: "3.6"
       SKIP_CI_IMAGE_CHECK: "true"
       RUNTIME: "kubernetes"
       ENABLE_KIND_CLUSTER: "true"
+      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
       KUBERNETES_MODE: "${{ matrix.kube-mode }}"
       KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
+    # For pull requests only run tests when python files changed
+    if: needs.pyfiles.outputs.count != '0' || github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@master
       - uses: actions/setup-python@v1
         with:
           python-version: '3.x'
       - name: "Free space"
         run: ./scripts/ci/ci_free_space_on_ci.sh
-      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
       - name: "Build PROD image ${{ matrix.python-version }}"
         run: ./scripts/ci/ci_build_production_images.sh
+      - name: "Setup KinD cluster"
+        run: ./scripts/ci/ci_perform_kind_cluster_operation.sh start
+      - name: "Deploy app to cluster"
+        run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh
+      - name: Cache virtualenv for kubernetes testing
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-kubernetes-tests-virtualenv-v2
+        with:
+          path: .build/.kubernetes_venv
+          key: "${{ env.cache-name }}-${{ github.job }}-\
+${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/ci_run_kubernetes_tests.sh
+
   tests-postgres:
     timeout-minutes: 80
     name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"

.gitignore

@@ -153,8 +153,8 @@ rat-results.txt
 # Kubernetes generated templated files
 *.generated
 *.tar.gz
-scripts/ci/in_container/kubernetes/kube/.generated/airflow.yaml
-scripts/ci/in_container/kubernetes/docker/requirements.txt
+scripts/ci/kubernetes/kube/.generated/airflow.yaml
+scripts/ci/kubernetes/docker/requirements.txt
 
 # Node & Webpack Stuff
 *.entry.js

.pre-commit-config.yaml

@@ -340,7 +340,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
         language: system
         entry: "./scripts/ci/pre_commit_pylint_main.sh"
         files: \.py$
-        exclude: ^tests/.*\.py$|^airflow/_vendor/.*|^scripts/.*\.py$|^dev|^backport_packages
+        exclude: ^tests/.*\.py$|^airflow/_vendor/.*|^scripts/.*\.py$|^dev|^backport_packages|^kubernetes_tests
         pass_filenames: true
         require_serial: true  # Pylint tests should be run in one chunk to detect all cycles
       - id: pylint-tests