Qubole as plugin (#557)

This deploys the changes from flyteorg/flytepropeller#200 lyft/flytedynamicjoboperator_DEPRECATED#98 flyteorg/flyteplugins#66 and changes the execution of the `hive` task type from the Qubole operator, to a plugin. It also add a pretty simple Redis deployment to use as a temporary FRM. **Note** that it currently uses and `emptyDir`, which means that if the Redis pod goes down, and it comes back up, we could potentially double the number of queries we want to run. If it goes down again, we could potentially triple. The yaml file for the redis deployment came from a local Helm deployment of redis. I then did an `-o yaml` get and heavily modified it and removed a bunch of stuff that wasn't needed. This PR does not remove the deployment of the Qubole operator yet, so we'll have to do that in a future PR, and the QuboleJob CRD will continue to function. Nothing will be using it though, so it should be okay.
flyteorg · Apr 9, 2019 · e85e21c · e85e21c
1 parent 7492b81
commit e85e21c
Show file tree

Hide file tree

Showing 36 changed files with 767 additions and 57 deletions.
diff --git a/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml b/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml
@@ -100,7 +100,7 @@ spec:
       labels:
         app: dynamic-job-operator
         app.kubernetes.io/name: dynamic-job-operator
-        app.kubernetes.io/version: 0.2.18
+        app.kubernetes.io/version: 0.2.19
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "10254"
@@ -130,7 +130,7 @@ spec:
             path: config.yaml
       containers:
       - name: dynamic-job-operator-unknown
-        image: docker.io/lyft/flytedynamicjoboperator:v0.2.18
+        image: docker.io/lyft/flytedynamicjoboperator:v0.2.19
         command:
         - flytedynamicjoboperator
         args:

diff --git a/artifacts/base/propeller/propeller.yaml b/artifacts/base/propeller/propeller.yaml
@@ -105,7 +105,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.37
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "10254"
@@ -135,7 +135,7 @@ spec:
             path: config.yaml
       containers:
       - name: flytepropeller
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.37
         command:
         - flytepropeller
         args:

diff --git a/artifacts/dependencies/redis/deployment.yaml b/artifacts/dependencies/redis/deployment.yaml
@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: redis-resource-manager
+  name: redis-resource-manager
+  namespace: flyte
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: redis-resource-manager
+  template:
+    metadata:
+      labels:
+        app: redis-resource-manager
+    spec:
+      containers:
+      - env:
+        - name: REDIS_PASSWORD
+          value: mypassword
+        image: bitnami/redis:4.0.2-r1
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: redis-resource-manager
+        ports:
+        - containerPort: 6379
+          name: redis
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 200m
+            memory: 512Mi
+        volumeMounts:
+        - mountPath: /bitnami
+          name: redis-data
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      volumes:
+      - name: redis-data
+        emptyDir: {}
diff --git a/artifacts/dependencies/redis/kustomization.yaml b/artifacts/dependencies/redis/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- deployment.yaml
+- service.yaml
diff --git a/artifacts/dependencies/redis/service.yaml b/artifacts/dependencies/redis/service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: redis-resource-manager
+  name: redis-resource-manager
+  namespace: flyte
+spec:
+  ports:
+  - name: redis
+    port: 6379
+    protocol: TCP
+    targetPort: redis
+  selector:
+    app: redis-resource-manager
+  type: ClusterIP
diff --git a/artifacts/development/flyte_generated.yaml b/artifacts/development/flyte_generated.yaml
@@ -1748,7 +1748,7 @@ data:
   common: |-
     logger:
       show-source: true
-      level: 5
+      level: 6
     storage:
       connection:
         access-key: minio
@@ -1796,8 +1796,6 @@ data:
       enabled-plugins:
       - K8S-ARRAY
       task-registry: admin
-    plugins:
-      debug-mode: true
     aws:
       cache-size: 10Mi
       region: us-east-1
@@ -1825,9 +1823,9 @@ data:
       enabled-plugins:
         - container
         - spark
-        - hive
         - waitable
         - K8S-ARRAY
+        - hiveExecutor
       logs:
         kubernetes-enabled: true
         kubernetes-url: "http://localhost:30082"
@@ -1924,6 +1922,24 @@ metadata:
   namespace: flyte
 ---
 apiVersion: v1
+data:
+  qubolePluginConfig: |-
+    plugins:
+      qubole:
+        # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable
+        # See the secrets_manager.go file in the plugins repo for usage.  Since the dev/test deployment of
+        # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything.
+        quboleTokenPath: "/Users/yourusername/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN"
+        resourceManagerType: redis
+        redisHostPath: redis-resource-manager.flyte:6379
+        redisHostKey: mypassword
+        quboleLimit: 10
+kind: ConfigMap
+metadata:
+  name: flyte-qubole-config
+  namespace: flyte
+---
+apiVersion: v1
 data:
   spark: |-
     plugins:
@@ -2066,6 +2082,23 @@ spec:
 ---
 apiVersion: v1
 kind: Service
+metadata:
+  labels:
+    app: redis-resource-manager
+  name: redis-resource-manager
+  namespace: flyte
+spec:
+  ports:
+  - name: redis
+    port: 6379
+    protocol: TCP
+    targetPort: redis
+  selector:
+    app: redis-resource-manager
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
 metadata:
   labels:
     app: contour
@@ -2114,7 +2147,7 @@ spec:
       labels:
         app: dynamic-job-operator
         app.kubernetes.io/name: dynamic-job-operator
-        app.kubernetes.io/version: 0.2.18
+        app.kubernetes.io/version: 0.2.19
     spec:
       containers:
       - args:
@@ -2123,7 +2156,10 @@ spec:
         - /etc/flyte/config*/config.yaml
         command:
         - flytedynamicjoboperator
-        image: docker.io/lyft/flytedynamicjoboperator:v0.2.18
+        env:
+        - name: QUBOLE_API_KEY
+          value: notarealkey
+        image: docker.io/lyft/flytedynamicjoboperator:v0.2.19
         imagePullPolicy: IfNotPresent
         name: dynamic-job-operator-unknown
         ports:
@@ -2134,6 +2170,8 @@ spec:
             ephemeral-storage: 100Mi
             memory: 100Mi
         volumeMounts:
+        - mountPath: /etc/flyte/config-qubole
+          name: qubole-config-volume
         - mountPath: /etc/flyte/config-container
           name: container-config-volume
         - mountPath: /etc/flyte/config-waitable
@@ -2150,6 +2188,12 @@ spec:
       - name: dockerhub
       serviceAccountName: dynamic-job-operator
       volumes:
+      - configMap:
+          items:
+          - key: qubolePluginConfig
+            path: config.yaml
+          name: flyte-qubole-config
+        name: qubole-config-volume
       - configMap:
           items:
           - key: container
@@ -2331,7 +2375,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.37
     spec:
       containers:
       - args:
@@ -2341,11 +2385,13 @@ spec:
         command:
         - flytepropeller
         env:
+        - name: QUBOLE_API_KEY
+          value: notarealkey
         - name: POD_NAME
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.37
         imagePullPolicy: Always
         name: flytepropeller
         ports:
@@ -2356,6 +2402,8 @@ spec:
             ephemeral-storage: 100Mi
             memory: 100Mi
         volumeMounts:
+        - mountPath: /etc/flyte/config-qubole
+          name: qubole-config-volume
         - mountPath: /etc/flyte/config-container
           name: container-config-volume
         - mountPath: /etc/flyte/config-waitable
@@ -2372,6 +2420,12 @@ spec:
       - name: dockerhub
       serviceAccountName: flytepropeller
       volumes:
+      - configMap:
+          items:
+          - key: qubolePluginConfig
+            path: config.yaml
+          name: flyte-qubole-config
+        name: qubole-config-volume
       - configMap:
           items:
           - key: container
@@ -2526,6 +2580,69 @@ spec:
 ---
 apiVersion: apps/v1
 kind: Deployment
+metadata:
+  labels:
+    app: redis-resource-manager
+  name: redis-resource-manager
+  namespace: flyte
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: redis-resource-manager
+  template:
+    metadata:
+      labels:
+        app: redis-resource-manager
+    spec:
+      containers:
+      - env:
+        - name: REDIS_PASSWORD
+          value: mypassword
+        image: bitnami/redis:4.0.2-r1
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: redis-resource-manager
+        ports:
+        - containerPort: 6379
+          name: redis
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 200m
+            memory: 512Mi
+        volumeMounts:
+        - mountPath: /bitnami
+          name: redis-data
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      volumes:
+      - emptyDir: {}
+        name: redis-data
+---
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   labels:
     app: contour

diff --git a/artifacts/overlays/development/common/configmap.yaml b/artifacts/overlays/development/common/configmap.yaml
@@ -7,7 +7,7 @@ data:
   common: |-
     logger:
       show-source: true
-      level: 5
+      level: 6
     storage:
       connection:
         access-key: minio

diff --git a/artifacts/overlays/development/dynamic-job-operator/configmap.yaml b/artifacts/overlays/development/dynamic-job-operator/configmap.yaml
@@ -14,8 +14,6 @@ data:
       enabled-plugins:
       - K8S-ARRAY
       task-registry: admin
-    plugins:
-      debug-mode: true
     aws:
       cache-size: 10Mi
       region: us-east-1

diff --git a/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml b/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml
@@ -12,9 +12,11 @@ spec:
     spec:
       containers:
       - name: dynamic-job-operator-unknown
+        env:
+        - name: QUBOLE_API_KEY
+          value: notarealkey
         resources:
           limits:
             cpu: "0.1"
             ephemeral-storage: 100Mi
             memory: 100Mi
-