github actions for k8s tests

.github/workflows/ci-k8s.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: "k8s tests"
+on: [pull_request, push]
+
+jobs:
+  create-airflow-dockerfile:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s-version: [v1.15.7, v1.16.4, v1.17.2, v1.18.0]
+    steps:
+      - uses: actions/checkout@v2
+        name: checkout
+      - name: create k8s Kind Cluster
+        uses: helm/[email protected]
+        with:
+          cluster_name: kind
+          node_image: kindest/node:${{ matrix.k8s-version }}
+          config: ./kind-cluster-conf.yaml
+      - name: test cluster running
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - name: download helm
+        run: |
+         curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+         chmod 700 get_helm.sh
+         ./get_helm.sh
+      - name: create namespace
+        run: kubectl create namespace airflow
+      - name: pull astronomer helm chart
+        run: |
+         helm repo add astronomer https://helm.astronomer.io
+         helm install airflow --namespace airflow astronomer/airflow \
+          --set env[0].name=AIRFLOW__CORE__LOAD_EXAMPLES \
+          --set env[0].value=True
+         kubectl get pods --namespace airflow
+         kubectl get svc --namespace airflow
+         kubectl port-forward --namespace airflow svc/airflow-webserver 8080:8080 &
+      - name: pip install airflow
+        run: pip install -e .[devel]
+      - name: run k8sexecutor tests
+        run: |
+         pytest tests/runtime/kubernetes/test_kubernetes_executor.py
+        env:
+          RUNTIME: kubernetes
+      - name: run k8sPodOperator tests
+        run: pytest tests/runtime/kubernetes/test_kubernetes_pod_operator.py
+        env:
+          RUNTIME: kubernetes

.travis.yml

@@ -55,26 +55,6 @@ jobs:
         PYTHON_MAJOR_MINOR_VERSION=3.6
       stage: test
       script: ./scripts/ci/ci_docs.sh
-    - name: "Tests [Py3.6][Kubernetes][persistent]"
-      env: >-
-        BACKEND=postgres
-        PYTHON_MAJOR_MINOR_VERSION=3.6
-        RUNTIME=kubernetes
-        ENABLE_KIND_CLUSTER=true
-        KUBERNETES_MODE=persistent_mode
-        KUBERNETES_VERSION=v1.15.3
-      python: "3.6"
-      stage: test
-    - name: "Tests [Py3.7][Kubernetes][git]"
-      env: >-
-        BACKEND=postgres
-        PYTHON_MAJOR_MINOR_VERSION=3.7
-        RUNTIME=kubernetes
-        ENABLE_KIND_CLUSTER=true
-        KUBERNETES_MODE=git_mode
-        KUBERNETES_VERSION=v1.15.3
-      python: "3.6"
-      stage: test
     - name: "Tests [Postgres9.6][Py3.6][integrations]"
       env: >-
         BACKEND=postgres

scripts/ci/in_container/kubernetes/app/secrets.yaml → kind-cluster-conf.yaml

@@ -15,12 +15,22 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-apiVersion: v1
-kind: Secret
-metadata:
-  name: airflow-secrets
-type: Opaque
-data:
-  # The sql_alchemy_conn value is a base64 encoded representation of this connection string:
-  # postgresql+psycopg2://root:root@postgres-airflow:5432/airflow
-  sql_alchemy_conn: cG9zdGdyZXNxbCtwc3ljb3BnMjovL3Jvb3Q6cm9vdEBwb3N0Z3Jlcy1haXJmbG93OjU0MzIvYWlyZmxvdwo=
+kind: Cluster
+apiVersion: kind.sigs.k8s.io/v1alpha3
+networking:
+  apiServerAddress: 0.0.0.0
+  apiServerPort: 19090
+nodes:
+  - role: control-plane
+  - role: worker
+    extraPortMappings:
+      - containerPort: 30809
+        hostPort: 30809
+kubeadmConfigPatchesJson6902:
+  - group: kubeadm.k8s.io
+    version: v1beta2
+    kind: ClusterConfiguration
+    patch: |
+      - op: add
+        path: /apiServer/certSANs/-
+        value: docker

scripts/ci/ci_run_airflow_testing.sh

@@ -78,37 +78,3 @@ do
 done
 
 RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""}
-
-if [[ ${RUNTIME:=} == "kubernetes" ]]; then
-    export KUBERNETES_MODE=${KUBERNETES_MODE:="git_mode"}
-    export KUBERNETES_VERSION=${KUBERNETES_VERSION:="v1.15.3"}
-
-    set +u
-    # shellcheck disable=SC2016
-    docker-compose --log-level INFO \
-      -f "${MY_DIR}/docker-compose/base.yml" \
-      -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \
-      -f "${MY_DIR}/docker-compose/runtime-kubernetes.yml" \
-      "${INTEGRATIONS[@]}" \
-      "${DOCKER_COMPOSE_LOCAL[@]}" \
-         run airflow \
-           '/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"' \
-           /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"
-         # Note the command is there twice (!) because it is passed via bash -c
-         # and bash -c starts passing parameters from $0. TODO: fixme
-    set -u
-else
-    set +u
-    # shellcheck disable=SC2016
-    docker-compose --log-level INFO \
-      -f "${MY_DIR}/docker-compose/base.yml" \
-      -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \
-      "${INTEGRATIONS[@]}" \
-      "${DOCKER_COMPOSE_LOCAL[@]}" \
-         run airflow \
-           '/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"' \
-           /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"
-         # Note the command is there twice (!) because it is passed via bash -c
-         # and bash -c starts passing parameters from $0. TODO: fixme
-    set -u
-fi