GKE single cluster configuration archetype

kustomize/overlays/gke-single-cluster/README.md

@@ -0,0 +1,48 @@
+###################################
+# WORK IN PROGRESS still
+###################################
+
+SQL Database
+------------
+Create a SQL database (Postgres)
+https://cloud.google.com/sql/docs/postgres/create-instance
+
+Enable the the SQL server to be accessed from the GKE cluster that will host the FlyteAdmin service. This can be done using private networking mode and associating the shared network
+
+Create a database called "flyte" in this DB instance
+
+Configuring Flyte to access DB
+------------------------------
+
+In this sample we pass the username and password directly in the config file.
+TODO: Example of how to use kube secrets to pass the username and password.
+
+Auth / IAM
+----------
+
+On GKE you can follow instructions listed here
+https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
+to setup WorkloadIdentity and serviceAccounts.
+
+Important commands
+kubectl create serviceaccount --namespace flytekit-development flyte-sandbox
+gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flytekit-development/flyte-sandbox]" [email protected]
+kubectl annotate serviceaccount  --namespace flytekit-development flyte-sandbox iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com
+
+
+IAM For FLyte components
+------------------------
+Create the right service accounts in GKE cluster's flyte namespace and then add the serviceaccountname to propeller and flyteadmin deployments. You may also want to add it to the various plugin
+deployments.
+
+gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flyte/flyteadmin]" [email protected]
+kubectl annotate serviceaccount  --namespace flyte flyteadmin iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com
+gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flyte/flytepropeller]" [email protected]
+kubectl annotate serviceaccount  --namespace flyte flytepropeller iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com
+
+IAM for workflows
+-----------------
+As a platform admin, you will need to associate service accounts with the target namespaces (project-domain) combination.  Flyte allows launching workflows with serviceAccounts. Thus when the end user
+requests a workflow launch or declares a workflow the right account should be associated within the right namespace. 
+
+TODO: Future plans to automate this creation and association

kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flyteadmin
+  namespace: flyte
+spec:
+  template:
+    spec:
+      volumes:
+      - name: resource-templates 
+        configMap:
+          name: clusterresource-template 
+      initContainers:
+      - name: check-db-ready
+        image: postgres:10.1
+        command: ['sh', '-c',
+          'until pg_isready -h postgres -p 5432;
+          do echo waiting for database; sleep 2; done;']
+      - name: run-migrations
+        image: docker.io/lyft/flyteadmin:v0.1.1
+        imagePullPolicy: IfNotPresent
+        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml",
+                  "migrate", "run"]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/flyte/config
+      - name: seed-projects
+        image: docker.io/lyft/flyteadmin:v0.1.1
+        imagePullPolicy: IfNotPresent
+        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml",
+                  "migrate", "seed-projects", "flytesnacks", "flytetester"]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/flyte/config
+      - name: sync-cluster-resources
+        image: docker.io/lyft/flyteadmin:v0.1.1
+        imagePullPolicy: IfNotPresent
+        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"]
+        volumeMounts:
+        - name: resource-templates 
+          mountPath: /etc/flyte/clusterresource/templates
+        - name: config-volume
+          mountPath: /etc/flyte/config
+      containers:
+      - name: flyteadmin
+        resources:
+          limits:
+            memory: "200Mi"
+            cpu: "0.1"
+            ephemeral-storage: "100Mi"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flyteadmin
+  namespace: flyte
+spec:
+  ports:
+  - name: redoc
+    protocol: TCP
+    port: 87
+    targetPort: 8087

kustomize/overlays/gke-single-cluster/admindeployment/clusterresource-templates/aa_namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ namespace }}
+spec:
+  finalizers:
+  - kubernetes

kustomize/overlays/gke-single-cluster/admindeployment/cron.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: syncresources
+  namespace: flyte
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: flyteadmin
+          containers:
+          - name: sync-cluster-resources
+            image: docker.io/lyft/flyteadmin:v0.1.1
+            imagePullPolicy: IfNotPresent
+            command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"]
+            volumeMounts:
+            - name: resource-templates 
+              mountPath: /etc/flyte/clusterresource/templates
+            - name: config-volume
+              mountPath: /etc/flyte/config
+          volumes:
+          - name: resource-templates 
+            configMap:
+              name: clusterresource-template 
+          - name: config-volume
+            configMap:
+              name: flyte-admin-config
+          restartPolicy: OnFailure

kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml

@@ -0,0 +1,86 @@
+logger:
+  show-source: true
+  level: 5
+application:
+  httpPort: 8088
+  grpcPort: 8089
+flyteadmin:
+  roleNameKey: "iam.amazonaws.com/role"
+  profilerPort: 10254
+  metricsScope: "flyte:"
+  metadataStoragePrefix:
+    - "metadata"
+    - "admin"
+  testing:
+    host: http://flyteadmin
+database:
+  # Create a database like postgres and override these values
+  port: 5432
+  username: postgres
+  password: awesomesauce
+  # Recommended to use passwordPath and mount it using kubescrets or the like
+  # passwordPath: "/var/run/CREDENTIALS_DB_PASSWORD"
+  # host here is the ip address of the CloudSQL Db in private mode
+  host: 10.23.0.3
+  dbname: flyte
+  options: sslmode=disable
+storage:
+  type: stow
+  stow:
+    kind: google
+    config:
+      scopes: ""
+      project_id: flyte-sandbox
+      json: ""
+  container: "flyte-sandbox"
+task_resources:
+  defaults:
+    cpu: 200m
+    gpu: 0
+    memory: 500Mi
+    storage: 100Mi
+  limits:
+    cpu: 62
+    gpu: 8
+    memory: 256Gi
+    storage: 5Gi
+domains:
+  - id: development
+    name: development
+  - id: staging
+    name: staging
+  - id: production
+    name: production
+  - id: domain
+    name: domain
+registration:
+  maxWorkflowNodes: 100
+scheduler:
+  eventScheduler:
+    scheme: local
+    region: "us-east-1"
+    scheduleRole: "arn:aws:iam::173840052742:role/mbadmin-development-scheduler"
+    targetName: "arn:aws:sqs:us-east-1:173840052742:flyteadmin-development-scheduler"
+  workflowExecutor:
+    scheme: local
+    region: "us-east-1"
+    scheduleQueueName: "won't-work-locally"
+    accountId: "173840052742"
+notifications:
+  type: local
+  region: "us-east-1"
+  publisher:
+    topicName: "foo"
+  processor:
+    queueName: "queue"
+    accountId: "bar"
+  emailer:
+    subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"."
+    sender:  "[email protected]"
+    body: >
+      Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at
+      <a href=\http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}>
+      http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}</a>. {{ error }}
+cluster_resources:
+  templatePath: "/etc/flyte/clusterresource/templates"
+  refresh: 5m

kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml

@@ -0,0 +1,22 @@
+bases:
+- ../../../base/admindeployment
+
+namespace: flyte
+
+resources:
+- cron.yaml
+- service.yaml
+
+configMapGenerator:
+# the main admin configmap
+- name: flyte-admin-config
+  files:
+    - flyteadmin_config.yaml
+# cluster resource templates
+- name: clusterresource-template
+  files:
+# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa".
+    - clusterresource-templates/aa_namespace.yaml
+
+patches:
+- admindeployment.yaml

kustomize/overlays/gke-single-cluster/admindeployment/service.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flyteadmin
+  annotations:
+    cloud.google.com/load-balancer-type: "Internal"
+spec:
+  type: LoadBalancer

kustomize/overlays/gke-single-cluster/console/console.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flyteconsole
+  namespace: flyte
+spec:
+  template:
+    spec:
+      containers:
+      - name: flyteconsole
+        resources:
+          limits:
+            memory: "150Mi"
+            cpu: "0.1"
+            ephemeral-storage: "100Mi"

kustomize/overlays/gke-single-cluster/console/kustomization.yaml

@@ -0,0 +1,8 @@
+bases:
+- ../../../base/console
+
+patches:
+- console.yaml
+
+resources:
+- service.yaml

kustomize/overlays/gke-single-cluster/console/service.yaml

@@ -0,0 +1,11 @@
+---
+# Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: flyteconsole
+  namespace: flyte
+  annotations:
+    cloud.google.com/load-balancer-type: "Internal"
+spec:
+  type: LoadBalancer

kustomize/overlays/gke-single-cluster/datacatalog/datacatalog.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: datacatalog 
+  namespace: flyte
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: check-db-ready
+        image: postgres:10.1
+        command: ['sh', '-c',
+          'until pg_isready -h postgres -p 5432;
+          do echo waiting for database; sleep 2; done;']
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/datacatalog/config
+      containers:
+      - name: datacatalog 
+        resources:
+          limits:
+            memory: "200Mi"
+            cpu: "0.1"
+            ephemeral-storage: "100Mi"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: datacatalog 
+  namespace: flyte

kustomize/overlays/gke-single-cluster/datacatalog/datacatalog_config.yaml

@@ -0,0 +1,28 @@
+logger:
+  show-source: true
+  level: 5
+datacatalog:
+  storage-prefix: metadata/datacatalog
+  metrics-scope: "datacatalog"
+  profiler-port: 10254
+application:
+  grpcPort: 8089
+storage:
+  connection:
+    access-key: minio
+    auth-type: accesskey
+    disable-ssl: true
+    endpoint: http://minio.flyte.svc.cluster.local:9000
+    region: us-east-1
+    secret-key: miniostorage
+  cache:
+    max_size_mbs: 10
+    target_gc_percent: 100
+  container: my-container
+  type: minio
+database:
+  port: 5432
+  username: postgres
+  host: postgres
+  dbname: datacatalog
+  options: sslmode=disable

kustomize/overlays/gke-single-cluster/datacatalog/kustomization.yaml

@@ -0,0 +1,12 @@
+bases:
+- ../../../base/datacatalog
+
+namespace: flyte
+
+configMapGenerator:
+- name: datacatalog-config
+  files:
+    - datacatalog_config.yaml
+
+patches:
+- datacatalog.yaml