From e85e21c0602945094ca6da84eb0b819203735e0a Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Tue, 9 Apr 2019 14:00:36 -0700 Subject: [PATCH] Qubole as plugin (#557) This deploys the changes from https://github.com/lyft/flytepropeller/pull/200 https://github.com/lyft/flytedynamicjoboperator/pull/98 https://github.com/lyft/flyteplugins/pull/66 and changes the execution of the `hive` task type from the Qubole operator, to a plugin. It also add a pretty simple Redis deployment to use as a temporary FRM. **Note** that it currently uses and `emptyDir`, which means that if the Redis pod goes down, and it comes back up, we could potentially double the number of queries we want to run. If it goes down again, we could potentially triple. The yaml file for the redis deployment came from a local Helm deployment of redis. I then did an `-o yaml` get and heavily modified it and removed a bunch of stuff that wasn't needed. This PR does not remove the deployment of the Qubole operator yet, so we'll have to do that in a future PR, and the QuboleJob CRD will continue to function. Nothing will be using it though, so it should be okay. --- .../dynamic-job-operator.yaml | 4 +- artifacts/base/propeller/propeller.yaml | 4 +- artifacts/dependencies/redis/deployment.yaml | 62 ++++++++ .../dependencies/redis/kustomization.yaml | 3 + artifacts/dependencies/redis/service.yaml | 16 +++ artifacts/development/flyte_generated.yaml | 133 ++++++++++++++++-- .../development/common/configmap.yaml | 2 +- .../dynamic-job-operator/configmap.yaml | 2 - .../dynamic-job-operator.yaml | 4 +- .../development/flyte/kustomization.yaml | 4 +- .../development/plugins/configmap.yaml | 2 +- .../development/plugins/kustomization.yaml | 3 +- .../development/plugins/qubole/configmap.yaml | 17 +++ .../plugins/qubole/kustomization.yaml | 2 + .../development/propeller/propeller.yaml | 3 + .../development/qubole-operator/config.yaml | 1 - .../dynamic-job-operator.yaml | 3 + .../production/flyte/kustomization.yaml | 5 +- .../production/plugins/kustomization.yaml | 3 +- .../production/plugins/qubole/configmap.yaml | 14 ++ .../plugins/qubole/kustomization.yaml | 2 + .../overlays/production/redis/deployment.yaml | 13 ++ .../overlays/staging/flyte/kustomization.yaml | 5 +- .../staging/plugins/kustomization.yaml | 3 +- .../staging/plugins/qubole/configmap.yaml | 14 ++ .../staging/plugins/qubole/kustomization.yaml | 2 + .../overlays/staging/redis/deployment.yaml | 13 ++ .../overlays/test/flyte/kustomization.yaml | 5 +- .../test/plugins/qubole/configmap.yaml | 13 ++ .../plugins/qubole/config-volumemount.yaml | 43 ++++++ artifacts/test/flyte_generated.yaml | 33 +++-- .../environment/critical/flyte_generated.yaml | 10 +- ops/k8s/environment/l5/flyte_generated.yaml | 8 +- .../environment/local/flyte_generated.yaml | 133 ++++++++++++++++-- .../production/flyte_generated.yaml | 121 +++++++++++++++- .../environment/staging/flyte_generated.yaml | 119 +++++++++++++++- 36 files changed, 767 insertions(+), 57 deletions(-) create mode 100644 artifacts/dependencies/redis/deployment.yaml create mode 100644 artifacts/dependencies/redis/kustomization.yaml create mode 100644 artifacts/dependencies/redis/service.yaml create mode 100644 artifacts/overlays/development/plugins/qubole/configmap.yaml create mode 100644 artifacts/overlays/development/plugins/qubole/kustomization.yaml create mode 100644 artifacts/overlays/production/plugins/qubole/configmap.yaml create mode 100644 artifacts/overlays/production/plugins/qubole/kustomization.yaml create mode 100644 artifacts/overlays/production/redis/deployment.yaml create mode 100644 artifacts/overlays/staging/plugins/qubole/configmap.yaml create mode 100644 artifacts/overlays/staging/plugins/qubole/kustomization.yaml create mode 100644 artifacts/overlays/staging/redis/deployment.yaml create mode 100644 artifacts/overlays/test/plugins/qubole/configmap.yaml create mode 100644 artifacts/plugins/qubole/config-volumemount.yaml diff --git a/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml b/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml index 16b9a6bf22a..bf7e62d058b 100644 --- a/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml +++ b/artifacts/base/dynamic-job-operator/dynamic-job-operator.yaml @@ -100,7 +100,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -130,7 +130,7 @@ spec: path: config.yaml containers: - name: dynamic-job-operator-unknown - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 command: - flytedynamicjoboperator args: diff --git a/artifacts/base/propeller/propeller.yaml b/artifacts/base/propeller/propeller.yaml index 400767f9d34..5614ad3b495 100644 --- a/artifacts/base/propeller/propeller.yaml +++ b/artifacts/base/propeller/propeller.yaml @@ -105,7 +105,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -135,7 +135,7 @@ spec: path: config.yaml containers: - name: flytepropeller - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 command: - flytepropeller args: diff --git a/artifacts/dependencies/redis/deployment.yaml b/artifacts/dependencies/redis/deployment.yaml new file mode 100644 index 00000000000..775b80e8f8d --- /dev/null +++ b/artifacts/dependencies/redis/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 512Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: redis-data + emptyDir: {} diff --git a/artifacts/dependencies/redis/kustomization.yaml b/artifacts/dependencies/redis/kustomization.yaml new file mode 100644 index 00000000000..a944d005ca7 --- /dev/null +++ b/artifacts/dependencies/redis/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- service.yaml diff --git a/artifacts/dependencies/redis/service.yaml b/artifacts/dependencies/redis/service.yaml new file mode 100644 index 00000000000..8c86264ae37 --- /dev/null +++ b/artifacts/dependencies/redis/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP diff --git a/artifacts/development/flyte_generated.yaml b/artifacts/development/flyte_generated.yaml index 9942f6ac3e1..4dd59fd04a8 100644 --- a/artifacts/development/flyte_generated.yaml +++ b/artifacts/development/flyte_generated.yaml @@ -1748,7 +1748,7 @@ data: common: |- logger: show-source: true - level: 5 + level: 6 storage: connection: access-key: minio @@ -1796,8 +1796,6 @@ data: enabled-plugins: - K8S-ARRAY task-registry: admin - plugins: - debug-mode: true aws: cache-size: 10Mi region: us-east-1 @@ -1825,9 +1823,9 @@ data: enabled-plugins: - container - spark - - hive - waitable - K8S-ARRAY + - hiveExecutor logs: kubernetes-enabled: true kubernetes-url: "http://localhost:30082" @@ -1924,6 +1922,24 @@ metadata: namespace: flyte --- apiVersion: v1 +data: + qubolePluginConfig: |- + plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Users/yourusername/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 +kind: ConfigMap +metadata: + name: flyte-qubole-config + namespace: flyte +--- +apiVersion: v1 data: spark: |- plugins: @@ -2066,6 +2082,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: labels: app: contour @@ -2114,7 +2147,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 spec: containers: - args: @@ -2123,7 +2156,10 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + env: + - name: QUBOLE_API_KEY + value: notarealkey + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: @@ -2134,6 +2170,8 @@ spec: ephemeral-storage: 100Mi memory: 100Mi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2150,6 +2188,12 @@ spec: - name: dockerhub serviceAccountName: dynamic-job-operator volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2331,7 +2375,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -2341,11 +2385,13 @@ spec: command: - flytepropeller env: + - name: QUBOLE_API_KEY + value: notarealkey - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: @@ -2356,6 +2402,8 @@ spec: ephemeral-storage: 100Mi memory: 100Mi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2372,6 +2420,12 @@ spec: - name: dockerhub serviceAccountName: flytepropeller volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2526,6 +2580,69 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 512Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: contour diff --git a/artifacts/overlays/development/common/configmap.yaml b/artifacts/overlays/development/common/configmap.yaml index 26a0d0b0695..35e20b6627f 100644 --- a/artifacts/overlays/development/common/configmap.yaml +++ b/artifacts/overlays/development/common/configmap.yaml @@ -7,7 +7,7 @@ data: common: |- logger: show-source: true - level: 5 + level: 6 storage: connection: access-key: minio diff --git a/artifacts/overlays/development/dynamic-job-operator/configmap.yaml b/artifacts/overlays/development/dynamic-job-operator/configmap.yaml index 18dedf91333..63f782b14d0 100644 --- a/artifacts/overlays/development/dynamic-job-operator/configmap.yaml +++ b/artifacts/overlays/development/dynamic-job-operator/configmap.yaml @@ -14,8 +14,6 @@ data: enabled-plugins: - K8S-ARRAY task-registry: admin - plugins: - debug-mode: true aws: cache-size: 10Mi region: us-east-1 diff --git a/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml b/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml index d59b2bd1204..22319474492 100644 --- a/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml +++ b/artifacts/overlays/development/dynamic-job-operator/dynamic-job-operator.yaml @@ -12,9 +12,11 @@ spec: spec: containers: - name: dynamic-job-operator-unknown + env: + - name: QUBOLE_API_KEY + value: notarealkey resources: limits: cpu: "0.1" ephemeral-storage: 100Mi memory: 100Mi - diff --git a/artifacts/overlays/development/flyte/kustomization.yaml b/artifacts/overlays/development/flyte/kustomization.yaml index 03435b28494..d8713562531 100644 --- a/artifacts/overlays/development/flyte/kustomization.yaml +++ b/artifacts/overlays/development/flyte/kustomization.yaml @@ -14,10 +14,12 @@ bases: - ../admindeployment - ../console - ../../../dependencies/contour_ingress_controller +- ../../../dependencies/redis - ../../../base/ingress - ../plugins patches: - ../../../plugins/spark/config-volumemount.yaml - ../../../plugins/waitable/config-volumemount.yaml -- ../../../plugins/container/config-volumemount.yaml \ No newline at end of file +- ../../../plugins/container/config-volumemount.yaml +- ../../../plugins/qubole/config-volumemount.yaml diff --git a/artifacts/overlays/development/plugins/configmap.yaml b/artifacts/overlays/development/plugins/configmap.yaml index 9a8a4e5f250..6dd17a85a69 100644 --- a/artifacts/overlays/development/plugins/configmap.yaml +++ b/artifacts/overlays/development/plugins/configmap.yaml @@ -10,9 +10,9 @@ data: enabled-plugins: - container - spark - - hive - waitable - K8S-ARRAY + - hiveExecutor logs: kubernetes-enabled: true kubernetes-url: "http://localhost:30082" diff --git a/artifacts/overlays/development/plugins/kustomization.yaml b/artifacts/overlays/development/plugins/kustomization.yaml index a7cc098ce42..52544c9f2b0 100644 --- a/artifacts/overlays/development/plugins/kustomization.yaml +++ b/artifacts/overlays/development/plugins/kustomization.yaml @@ -4,4 +4,5 @@ resources: bases: - ../plugins/spark - ../plugins/waitable -- ../../../plugins/container \ No newline at end of file +- ../plugins/qubole +- ../../../plugins/container diff --git a/artifacts/overlays/development/plugins/qubole/configmap.yaml b/artifacts/overlays/development/plugins/qubole/configmap.yaml new file mode 100644 index 00000000000..93f40ed1c19 --- /dev/null +++ b/artifacts/overlays/development/plugins/qubole/configmap.yaml @@ -0,0 +1,17 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-qubole-config + namespace: flyte +data: + qubolePluginConfig: |- + plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Users/yourusername/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 diff --git a/artifacts/overlays/development/plugins/qubole/kustomization.yaml b/artifacts/overlays/development/plugins/qubole/kustomization.yaml new file mode 100644 index 00000000000..f8c7f8f7a5e --- /dev/null +++ b/artifacts/overlays/development/plugins/qubole/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- configmap.yaml diff --git a/artifacts/overlays/development/propeller/propeller.yaml b/artifacts/overlays/development/propeller/propeller.yaml index 19dc542afc1..8e9f3b4c5b9 100644 --- a/artifacts/overlays/development/propeller/propeller.yaml +++ b/artifacts/overlays/development/propeller/propeller.yaml @@ -10,6 +10,9 @@ spec: spec: containers: - name: flytepropeller + env: + - name: QUBOLE_API_KEY + value: notarealkey resources: limits: memory: "100Mi" diff --git a/artifacts/overlays/development/qubole-operator/config.yaml b/artifacts/overlays/development/qubole-operator/config.yaml index 4b67c1bb164..daa1a85a820 100644 --- a/artifacts/overlays/development/qubole-operator/config.yaml +++ b/artifacts/overlays/development/qubole-operator/config.yaml @@ -31,4 +31,3 @@ data: logger: show-source: true level: 5 - diff --git a/artifacts/overlays/production/dynamic-job-operator/dynamic-job-operator.yaml b/artifacts/overlays/production/dynamic-job-operator/dynamic-job-operator.yaml index 67e2a759eef..65b9b0286f3 100644 --- a/artifacts/overlays/production/dynamic-job-operator/dynamic-job-operator.yaml +++ b/artifacts/overlays/production/dynamic-job-operator/dynamic-job-operator.yaml @@ -9,3 +9,6 @@ spec: metadata: annotations: iam.amazonaws.com/role: "arn:aws:iam::173840052742:role/flyteoperatorbatchworker-staging" + lyft.net/secret-inject: "required" + labels: + secretsIam: "flyteoperatorbatchworker-staging" diff --git a/artifacts/overlays/production/flyte/kustomization.yaml b/artifacts/overlays/production/flyte/kustomization.yaml index af007eebf09..32c6734744a 100644 --- a/artifacts/overlays/production/flyte/kustomization.yaml +++ b/artifacts/overlays/production/flyte/kustomization.yaml @@ -13,10 +13,13 @@ bases: - ../admindeployment - ../console - ../../../base/ingress +- ../../../dependencies/redis - ../stats - ../plugins patches: - ../../../plugins/spark/config-volumemount.yaml - ../../../plugins/waitable/config-volumemount.yaml -- ../../../plugins/container/config-volumemount.yaml \ No newline at end of file +- ../../../plugins/container/config-volumemount.yaml +- ../../../plugins/qubole/config-volumemount.yaml +- ../redis/deployment.yaml diff --git a/artifacts/overlays/production/plugins/kustomization.yaml b/artifacts/overlays/production/plugins/kustomization.yaml index a7cc098ce42..52544c9f2b0 100644 --- a/artifacts/overlays/production/plugins/kustomization.yaml +++ b/artifacts/overlays/production/plugins/kustomization.yaml @@ -4,4 +4,5 @@ resources: bases: - ../plugins/spark - ../plugins/waitable -- ../../../plugins/container \ No newline at end of file +- ../plugins/qubole +- ../../../plugins/container diff --git a/artifacts/overlays/production/plugins/qubole/configmap.yaml b/artifacts/overlays/production/plugins/qubole/configmap.yaml new file mode 100644 index 00000000000..4b959ed7296 --- /dev/null +++ b/artifacts/overlays/production/plugins/qubole/configmap.yaml @@ -0,0 +1,14 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-qubole-config + namespace: flyte +data: + qubolePluginConfig: |- + plugins: + qubole: + quboleTokenPath: "/var/run/confidant/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 200 diff --git a/artifacts/overlays/production/plugins/qubole/kustomization.yaml b/artifacts/overlays/production/plugins/qubole/kustomization.yaml new file mode 100644 index 00000000000..f8c7f8f7a5e --- /dev/null +++ b/artifacts/overlays/production/plugins/qubole/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- configmap.yaml diff --git a/artifacts/overlays/production/redis/deployment.yaml b/artifacts/overlays/production/redis/deployment.yaml new file mode 100644 index 00000000000..5d82cd23995 --- /dev/null +++ b/artifacts/overlays/production/redis/deployment.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-resource-manager +spec: + template: + spec: + containers: + - name: redis-resource-manager + resources: + requests: + cpu: "64" + memory: "400G" diff --git a/artifacts/overlays/staging/flyte/kustomization.yaml b/artifacts/overlays/staging/flyte/kustomization.yaml index 204f6f8607b..a8abf78f767 100644 --- a/artifacts/overlays/staging/flyte/kustomization.yaml +++ b/artifacts/overlays/staging/flyte/kustomization.yaml @@ -12,6 +12,7 @@ bases: - ../fluentd-logging - ../qubole-operator - ../admindeployment +- ../../../dependencies/redis # staging console is the same as prod - ../../production/console - ../stats @@ -20,4 +21,6 @@ bases: patches: - ../../../plugins/spark/config-volumemount.yaml - ../../../plugins/waitable/config-volumemount.yaml -- ../../../plugins/container/config-volumemount.yaml \ No newline at end of file +- ../../../plugins/container/config-volumemount.yaml +- ../../../plugins/qubole/config-volumemount.yaml +- ../redis/deployment.yaml diff --git a/artifacts/overlays/staging/plugins/kustomization.yaml b/artifacts/overlays/staging/plugins/kustomization.yaml index a7cc098ce42..52544c9f2b0 100644 --- a/artifacts/overlays/staging/plugins/kustomization.yaml +++ b/artifacts/overlays/staging/plugins/kustomization.yaml @@ -4,4 +4,5 @@ resources: bases: - ../plugins/spark - ../plugins/waitable -- ../../../plugins/container \ No newline at end of file +- ../plugins/qubole +- ../../../plugins/container diff --git a/artifacts/overlays/staging/plugins/qubole/configmap.yaml b/artifacts/overlays/staging/plugins/qubole/configmap.yaml new file mode 100644 index 00000000000..b0219ef6563 --- /dev/null +++ b/artifacts/overlays/staging/plugins/qubole/configmap.yaml @@ -0,0 +1,14 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-qubole-config + namespace: flyte +data: + qubolePluginConfig: |- + plugins: + qubole: + quboleTokenPath: "/var/run/confidant/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 diff --git a/artifacts/overlays/staging/plugins/qubole/kustomization.yaml b/artifacts/overlays/staging/plugins/qubole/kustomization.yaml new file mode 100644 index 00000000000..f8c7f8f7a5e --- /dev/null +++ b/artifacts/overlays/staging/plugins/qubole/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- configmap.yaml diff --git a/artifacts/overlays/staging/redis/deployment.yaml b/artifacts/overlays/staging/redis/deployment.yaml new file mode 100644 index 00000000000..5d82cd23995 --- /dev/null +++ b/artifacts/overlays/staging/redis/deployment.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-resource-manager +spec: + template: + spec: + containers: + - name: redis-resource-manager + resources: + requests: + cpu: "64" + memory: "400G" diff --git a/artifacts/overlays/test/flyte/kustomization.yaml b/artifacts/overlays/test/flyte/kustomization.yaml index 138092e964b..c0e95127845 100644 --- a/artifacts/overlays/test/flyte/kustomization.yaml +++ b/artifacts/overlays/test/flyte/kustomization.yaml @@ -17,4 +17,7 @@ bases: - ../admindeployment - ../../../dependencies/contour_ingress_controller - ../../../base/ingress -- ../../development/plugins \ No newline at end of file +- ../../development/plugins + +patches: +- ../plugins/qubole/configmap.yaml diff --git a/artifacts/overlays/test/plugins/qubole/configmap.yaml b/artifacts/overlays/test/plugins/qubole/configmap.yaml new file mode 100644 index 00000000000..48df05421fc --- /dev/null +++ b/artifacts/overlays/test/plugins/qubole/configmap.yaml @@ -0,0 +1,13 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-qubole-config + namespace: flyte +data: + qubolePluginConfig: |- + plugins: + qubole: + # For testing, we should just use the noop resource manager. Qubole tasks don't run in these kinds of tests + # anyways, at least not yet, so it doesn't really matter. + resourceManagerType: noop + quboleLimit: 10 # but not needed since tests only use noop diff --git a/artifacts/plugins/qubole/config-volumemount.yaml b/artifacts/plugins/qubole/config-volumemount.yaml new file mode 100644 index 00000000000..33ca1c77dad --- /dev/null +++ b/artifacts/plugins/qubole/config-volumemount.yaml @@ -0,0 +1,43 @@ +# This file is only for volume mounts. The configmap itself that's being mounted is sufficiently different that +# there's no benefit to having it in this folder, since the entire thing gets overridden anyways. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller +spec: + template: + spec: + volumes: + - name: qubole-config-volume + configMap: + name: flyte-qubole-config + items: + - key: qubolePluginConfig + path: config.yaml + containers: + - name: flytepropeller + volumeMounts: + - name: qubole-config-volume + mountPath: /etc/flyte/config-qubole + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dynamic-job-operator +spec: + template: + spec: + volumes: + - name: qubole-config-volume + configMap: + name: flyte-qubole-config + items: + - key: qubolePluginConfig + path: config.yaml + containers: + - name: dynamic-job-operator-unknown + volumeMounts: + - name: qubole-config-volume + mountPath: /etc/flyte/config-qubole diff --git a/artifacts/test/flyte_generated.yaml b/artifacts/test/flyte_generated.yaml index 39615f3ffca..6f6dc1a0a65 100644 --- a/artifacts/test/flyte_generated.yaml +++ b/artifacts/test/flyte_generated.yaml @@ -1748,7 +1748,7 @@ data: common: |- logger: show-source: true - level: 5 + level: 6 storage: connection: access-key: minio @@ -1787,8 +1787,6 @@ data: enabled-plugins: - K8S-ARRAY task-registry: admin - plugins: - debug-mode: true aws: cache-size: 10Mi region: us-east-1 @@ -1816,9 +1814,9 @@ data: enabled-plugins: - container - spark - - hive - waitable - K8S-ARRAY + - hiveExecutor logs: kubernetes-enabled: true kubernetes-url: "http://localhost:30082" @@ -1915,6 +1913,20 @@ metadata: namespace: flyte --- apiVersion: v1 +data: + qubolePluginConfig: |- + plugins: + qubole: + # For testing, we should just use the noop resource manager. Qubole tasks don't run in these kinds of tests + # anyways, at least not yet, so it doesn't really matter. + resourceManagerType: noop + quboleLimit: 10 # but not needed since tests only use noop +kind: ConfigMap +metadata: + name: flyte-qubole-config + namespace: flyte +--- +apiVersion: v1 data: spark: |- plugins: @@ -2092,7 +2104,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 spec: containers: - args: @@ -2101,7 +2113,10 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + env: + - name: QUBOLE_API_KEY + value: notarealkey + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: @@ -2246,7 +2261,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -2256,11 +2271,13 @@ spec: command: - flytepropeller env: + - name: QUBOLE_API_KEY + value: notarealkey - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: diff --git a/ops/k8s/environment/critical/flyte_generated.yaml b/ops/k8s/environment/critical/flyte_generated.yaml index 8727ddf004c..a067b438162 100644 --- a/ops/k8s/environment/critical/flyte_generated.yaml +++ b/ops/k8s/environment/critical/flyte_generated.yaml @@ -1755,13 +1755,15 @@ spec: metadata: annotations: iam.amazonaws.com/role: arn:aws:iam::173840052742:role/flyteoperatorbatchworker-staging + lyft.net/secret-inject: required prometheus.io/path: /metrics prometheus.io/port: "10254" prometheus.io/scrape: "true" labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 + secretsIam: flyteoperatorbatchworker-staging spec: containers: - args: @@ -1770,7 +1772,7 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: @@ -1876,7 +1878,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -1890,7 +1892,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: diff --git a/ops/k8s/environment/l5/flyte_generated.yaml b/ops/k8s/environment/l5/flyte_generated.yaml index 6504dc18f99..a5a1915f684 100644 --- a/ops/k8s/environment/l5/flyte_generated.yaml +++ b/ops/k8s/environment/l5/flyte_generated.yaml @@ -1221,7 +1221,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 spec: containers: - args: @@ -1230,7 +1230,7 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: @@ -1394,7 +1394,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -1408,7 +1408,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: diff --git a/ops/k8s/environment/local/flyte_generated.yaml b/ops/k8s/environment/local/flyte_generated.yaml index 9942f6ac3e1..4dd59fd04a8 100644 --- a/ops/k8s/environment/local/flyte_generated.yaml +++ b/ops/k8s/environment/local/flyte_generated.yaml @@ -1748,7 +1748,7 @@ data: common: |- logger: show-source: true - level: 5 + level: 6 storage: connection: access-key: minio @@ -1796,8 +1796,6 @@ data: enabled-plugins: - K8S-ARRAY task-registry: admin - plugins: - debug-mode: true aws: cache-size: 10Mi region: us-east-1 @@ -1825,9 +1823,9 @@ data: enabled-plugins: - container - spark - - hive - waitable - K8S-ARRAY + - hiveExecutor logs: kubernetes-enabled: true kubernetes-url: "http://localhost:30082" @@ -1924,6 +1922,24 @@ metadata: namespace: flyte --- apiVersion: v1 +data: + qubolePluginConfig: |- + plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Users/yourusername/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 +kind: ConfigMap +metadata: + name: flyte-qubole-config + namespace: flyte +--- +apiVersion: v1 data: spark: |- plugins: @@ -2066,6 +2082,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: labels: app: contour @@ -2114,7 +2147,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 spec: containers: - args: @@ -2123,7 +2156,10 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + env: + - name: QUBOLE_API_KEY + value: notarealkey + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: @@ -2134,6 +2170,8 @@ spec: ephemeral-storage: 100Mi memory: 100Mi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2150,6 +2188,12 @@ spec: - name: dockerhub serviceAccountName: dynamic-job-operator volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2331,7 +2375,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -2341,11 +2385,13 @@ spec: command: - flytepropeller env: + - name: QUBOLE_API_KEY + value: notarealkey - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: @@ -2356,6 +2402,8 @@ spec: ephemeral-storage: 100Mi memory: 100Mi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2372,6 +2420,12 @@ spec: - name: dockerhub serviceAccountName: flytepropeller volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2526,6 +2580,69 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 512Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: contour diff --git a/ops/k8s/environment/production/flyte_generated.yaml b/ops/k8s/environment/production/flyte_generated.yaml index 3781f58c881..b14a4bddf1d 100644 --- a/ops/k8s/environment/production/flyte_generated.yaml +++ b/ops/k8s/environment/production/flyte_generated.yaml @@ -1848,6 +1848,21 @@ metadata: namespace: flyte --- apiVersion: v1 +data: + qubolePluginConfig: |- + plugins: + qubole: + quboleTokenPath: "/var/run/confidant/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 200 +kind: ConfigMap +metadata: + name: flyte-qubole-config + namespace: flyte +--- +apiVersion: v1 data: spark: |- plugins: @@ -2001,6 +2016,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: name: spark-history-service namespace: sparkoperator @@ -2042,13 +2074,15 @@ spec: metadata: annotations: iam.amazonaws.com/role: arn:aws:iam::173840052742:role/flyteoperatorbatchworker-staging + lyft.net/secret-inject: required prometheus.io/path: /metrics prometheus.io/port: "10254" prometheus.io/scrape: "true" labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 + secretsIam: flyteoperatorbatchworker-staging spec: containers: - args: @@ -2057,12 +2091,14 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: - containerPort: 10254 volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2079,6 +2115,12 @@ spec: - name: dockerhub serviceAccountName: dynamic-job-operator volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2245,7 +2287,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -2259,7 +2301,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: @@ -2272,6 +2314,8 @@ spec: cpu: "4" memory: 4Gi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2288,6 +2332,12 @@ spec: - name: dockerhub serviceAccountName: flytepropeller volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2380,6 +2430,69 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: "64" + memory: 400G + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: apps/v1 +kind: Deployment metadata: annotations: iam.amazonaws.com/role: arn:aws:iam::173840052742:role/modelbuilderapibatchworker-development diff --git a/ops/k8s/environment/staging/flyte_generated.yaml b/ops/k8s/environment/staging/flyte_generated.yaml index f56a5aaf58f..d85c6863e41 100644 --- a/ops/k8s/environment/staging/flyte_generated.yaml +++ b/ops/k8s/environment/staging/flyte_generated.yaml @@ -1822,6 +1822,21 @@ metadata: namespace: flyte --- apiVersion: v1 +data: + qubolePluginConfig: |- + plugins: + qubole: + quboleTokenPath: "/var/run/confidant/CREDENTIALS_FLYTE_QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 +kind: ConfigMap +metadata: + name: flyte-qubole-config + namespace: flyte +--- +apiVersion: v1 data: spark: |- plugins: @@ -1971,6 +1986,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: name: spark-history-service namespace: sparkoperator @@ -2018,7 +2050,7 @@ spec: labels: app: dynamic-job-operator app.kubernetes.io/name: dynamic-job-operator - app.kubernetes.io/version: 0.2.18 + app.kubernetes.io/version: 0.2.19 spec: containers: - args: @@ -2027,12 +2059,14 @@ spec: - /etc/flyte/config*/config.yaml command: - flytedynamicjoboperator - image: docker.io/lyft/flytedynamicjoboperator:v0.2.18 + image: docker.io/lyft/flytedynamicjoboperator:v0.2.19 imagePullPolicy: IfNotPresent name: dynamic-job-operator-unknown ports: - containerPort: 10254 volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2049,6 +2083,12 @@ spec: - name: dockerhub serviceAccountName: dynamic-job-operator volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2215,7 +2255,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.37 spec: containers: - args: @@ -2229,7 +2269,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.37 imagePullPolicy: Always name: flytepropeller ports: @@ -2242,6 +2282,8 @@ spec: cpu: "4" memory: 4Gi volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume - mountPath: /etc/flyte/config-container name: container-config-volume - mountPath: /etc/flyte/config-waitable @@ -2258,6 +2300,12 @@ spec: - name: dockerhub serviceAccountName: flytepropeller volumes: + - configMap: + items: + - key: qubolePluginConfig + path: config.yaml + name: flyte-qubole-config + name: qubole-config-volume - configMap: items: - key: container @@ -2350,6 +2398,69 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: "64" + memory: 400G + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: apps/v1 +kind: Deployment metadata: annotations: iam.amazonaws.com/role: arn:aws:iam::173840052742:role/modelbuilderapibatchworker-development