From d3c1d904fbcdf5ac12bbec1dc953dcb2770e33f9 Mon Sep 17 00:00:00 2001 From: Yuvraj Date: Fri, 6 May 2022 17:51:19 +0530 Subject: [PATCH] more changes Signed-off-by: Yuvraj --- docker/sandbox/flyte-entrypoint-default.sh | 1 + rsts/deployment/plugin_setup/k8s/index.rst | 479 ++++++++++++------ .../plugin_setup/webapi/snowflake.rst | 25 +- 3 files changed, 346 insertions(+), 159 deletions(-) diff --git a/docker/sandbox/flyte-entrypoint-default.sh b/docker/sandbox/flyte-entrypoint-default.sh index 3630a72d8f7..21d4ea134ec 100755 --- a/docker/sandbox/flyte-entrypoint-default.sh +++ b/docker/sandbox/flyte-entrypoint-default.sh @@ -36,6 +36,7 @@ K3S_PID=$! timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) echo "Done." +echo "Deploying Flyte..." FLYTE_VERSION=${FLYTE_VERSION:-latest} if [[ $FLYTE_VERSION = "latest" ]] then diff --git a/rsts/deployment/plugin_setup/k8s/index.rst b/rsts/deployment/plugin_setup/k8s/index.rst index 4d100e1243d..696dfa33d6e 100644 --- a/rsts/deployment/plugin_setup/k8s/index.rst +++ b/rsts/deployment/plugin_setup/k8s/index.rst @@ -15,7 +15,23 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo 2. Setup the cluster - * Make sure you have up and running flyte cluster in `AWS `__ / `GCP `__ / `Sandbox `__ +.. tabbed:: Sandbox + + * Start the sandbox cluster + + .. code-block:: bash + + flytectl sandbox start + + * Generate Flytectl sandbox config + + .. code-block:: bash + + flytectl config init + +.. tabbed:: AWS/GCP + + * Make sure you have up and running flyte cluster in `AWS `__ / `GCP `__ * Make sure you have correct kubeconfig and selected the correct kubernetes context * make sure you have the correct flytectl config at ~/.flyte/config.yaml @@ -162,160 +178,308 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. tabbed:: Spark Operator - * Enable Spark backend plugin + .. tabbed:: Sandbox + + Since sandbox uses minio, it needs additional configuration. .. code-block:: yaml - cluster_resource_manager: - # -- Enables the Cluster resource manager component - enabled: true - # -- Configmap for ClusterResource parameters - config: - # -- ClusterResource parameters - # Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. - cluster_resources: - refreshInterval: 5m - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refresh: 5m - - # -- Resource templates that should be applied - templates: - # -- Template for namespaces resources - - key: aa_namespace - value: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - - - key: ab_project_resource_quota - value: | - apiVersion: v1 - kind: ResourceQuota - metadata: - name: project-quota - namespace: {{ namespace }} - spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} - - - key: ac_spark_role - value: | - apiVersion: rbac.authorization.k8s.io/v1beta1 + cluster_resource_manager: + # -- Enables the Cluster resource manager component + enabled: true + # -- Configmap for ClusterResource parameters + config: + # -- ClusterResource parameters + # Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. + cluster_resources: + refreshInterval: 5m + templatePath: "/etc/flyte/clusterresource/templates" + customData: + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "5000Mi" + refresh: 5m + + # -- Resource templates that should be applied + templates: + # -- Template for namespaces resources + - key: aa_namespace + value: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + - key: ab_project_resource_quota + value: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + - key: ac_spark_role + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: ["*"] + resources: ["pods"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["services"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["configmaps", "persistentvolumeclaims"] + verbs: ["*"] + + - key: ad_spark_service_account + value: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + + - key: ae_spark_role_binding + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - '*' - - apiGroups: - - "" - resources: - - services - verbs: - - '*' - - apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' - - - key: ad_spark_service_account - value: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - - - key: ae_spark_role_binding - value: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: {{ namespace }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: {{ namespace }} - - sparkoperator: - enabled: true - plugin_config: - plugins: - spark: - # -- Spark default configuration - spark-config-default: - # We override the default credentials chain provider for Hadoop so that - # it can use the serviceAccount based IAM role or ec2 metadata based. - # This is more in line with how AWS works - - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" - configmap: - enabled_plugins: - # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) - tasks: - # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) - task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend - # plugins - enabled-plugins: - - container - - sidecar - - k8s-array - - spark - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - spark: spark + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} + + sparkoperator: + enabled: true + plugin_config: + plugins: + spark: + # -- Spark default configuration + spark-config-default: + # We override the default credentials chain provider for Hadoop so that + # it can use the serviceAccount based IAM role or ec2 metadata based. + # This is more in line with how AWS works + - spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + - spark.hadoop.fs.s3a.endpoint: "http://minio.flyte.svc.cluster.local:9000" + - spark.hadoop.fs.s3a.access.key: "minio" + - spark.hadoop.fs.s3a.secret.key: "miniostorage" + - spark.hadoop.fs.s3a.path.style.access: "true" + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.excludeOnFailure.enabled: "true" + - spark.excludeOnFailure.timeout: "5m" + - spark.task.maxfailures: "8" + configmap: + enabled_plugins: + # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) + tasks: + # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) + task-plugins: + # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend + # plugins + enabled-plugins: + - container + - sidecar + - k8s-array + - spark + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array + spark: spark + + .. tabbed:: AWS + + .. code-block:: yaml + + cluster_resource_manager: + # -- Enables the Cluster resource manager component + enabled: true + # -- Configmap for ClusterResource parameters + config: + # -- ClusterResource parameters + # Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. + cluster_resources: + refreshInterval: 5m + templatePath: "/etc/flyte/clusterresource/templates" + customData: + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" + refresh: 5m + + # -- Resource templates that should be applied + templates: + # -- Template for namespaces resources + - key: aa_namespace + value: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + - key: ab_project_resource_quota + value: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + - key: ac_spark_role + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: ["*"] + resources: + - pods + verbs: + - '*' + - apiGroups: ["*"] + resources: + - services + verbs: + - '*' + - apiGroups: ["*"] + resources: + - configmaps + verbs: + - '*' + + - key: ad_spark_service_account + value: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + + - key: ae_spark_role_binding + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} + + sparkoperator: + enabled: true + plugin_config: + plugins: + spark: + # -- Spark default configuration + spark-config-default: + # We override the default credentials chain provider for Hadoop so that + # it can use the serviceAccount based IAM role or ec2 metadata based. + # This is more in line with how AWS works + - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.excludeOnFailure.enabled: "true" + - spark.excludeOnFailure.timeout: "5m" + - spark.task.maxfailures: "8" + configmap: + enabled_plugins: + # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) + tasks: + # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) + task-plugins: + # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend + # plugins + enabled-plugins: + - container + - sidecar + - k8s-array + - spark + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array + spark: spark 5. Upgrade the Flyte Helm release. - helm upgrade flyte-core flyteorg/flyte-core -f values-override.yaml -n flyte +.. code-block:: bash + + helm upgrade flyte flyteorg/flyte-core -f values-override.yaml -n flyte + 6. Register the plugin example. @@ -323,26 +487,26 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. code-block:: bash - flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.2.225/snacks-cookbook-integrations-kubernetes-kfpytorch.tar.gz --archive -p flytesnacks -d development --version latest + flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.3.75/snacks-cookbook-integrations-kubernetes-kfpytorch.tar.gz --archive -p flytesnacks -d development --version latest .. tabbed:: TensorFlow Operator .. code-block:: bash # TODO: https://github.com/flyteorg/flyte/issues/1757 - flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.2.225/.tar.gz --archive -p flytesnacks -d development --version latest + flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.3.75/snacks-cookbook-integrations-kubernetes-kftensorflow.tar.gz --archive -p flytesnacks -d development --version latest .. tabbed:: MPI Operator .. code-block:: bash - flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.2.226/snacks-cookbook-integrations-kubernetes-kfmpi.tar.gz --archive -p flytesnacks -d development --version latest + flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.3.75/snacks-cookbook-integrations-kubernetes-kfmpi.tar.gz --archive -p flytesnacks -d development --version latest .. tabbed:: Spark Operator .. code-block:: bash - flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.2.226/snacks-cookbook-integrations-kubernetes-k8s_spark.tar.gz --archive -p flytesnacks -d development --version latest + flytectl register files --config ~/.flyte/config.yaml https://github.com/flyteorg/flytesnacks/releases/download/v0.3.75/snacks-cookbook-integrations-kubernetes-k8s_spark.tar.gz --archive -p flytesnacks -d development --version latest 7. Launch an execution @@ -351,6 +515,7 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo * Navigate to the Flyte Console's UI (e.g. `sandbox `_) and find the relevant workflow * Click on `Launch` to open up a launch form + * Give **spark** as the service account if launching a spark example * Submit the form to launch an execution .. tabbed:: Flytectl @@ -361,7 +526,7 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. code-block:: bash - flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development kfpytorch.pytorch_mnist.pytorch_training_wf --latest --execFile exec_spec.yaml --config ~/.flyte/config.yaml + flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development kfpytorch.pytorch_mnist.pytorch_training_wf --latest --execFile exec_spec.yaml * Launch! 🚀 @@ -375,7 +540,7 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. code-block:: bash - flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development --latest --execFile exec_spec.yaml --config ~/.flyte/config.yaml + flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development --latest --execFile exec_spec.yaml * Launch! 🚀 @@ -389,7 +554,7 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. code-block:: bash - flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development kfmpi.mpi_mnist.horovod_training_wf --latest --execFile exec_spec.yaml --config ~/.flyte/config.yaml + flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development kfmpi.mpi_mnist.horovod_training_wf --latest --execFile exec_spec.yaml * Launch! 🚀 @@ -403,7 +568,9 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo .. code-block:: bash - flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development k8s_spark.pyspark_pi.my_spark --latest --execFile exec_spec.yaml --config ~/.flyte/config.yaml + flytectl get launchplan --config ~/.flyte/config.yaml --project flytesnacks --domain development k8s_spark.pyspark_pi.my_spark --latest --execFile exec_spec.yaml + + * Fill in the ``kubeServiceAcct`` as **spark** in the ``exec_spec.yaml`` file * Launch! 🚀 diff --git a/rsts/deployment/plugin_setup/webapi/snowflake.rst b/rsts/deployment/plugin_setup/webapi/snowflake.rst index c0b8c7896b0..4ff62e18654 100644 --- a/rsts/deployment/plugin_setup/webapi/snowflake.rst +++ b/rsts/deployment/plugin_setup/webapi/snowflake.rst @@ -11,9 +11,26 @@ This guide gives an overview of how to set up Snowflake in your Flyte deployment helm repo add flyteorg https://flyteorg.github.io/flyte + 2. Setup the cluster - * Make sure you have up and running flyte cluster in `AWS `__ / `GCP `__ / `Sandbox `__ +.. tabbed:: Sandbox + + * Start the sandbox cluster + + .. code-block:: bash + + flytectl sandbox start + + * Generate Flytectl sandbox config + + .. code-block:: bash + + flytectl config init + +.. tabbed:: AWS/GCP + + * Make sure you have up and running flyte cluster in `AWS `__ / `GCP `__ * Make sure you have correct kubeconfig and selected the correct kubernetes context * make sure you have the correct flytectl config at ~/.flyte/config.yaml @@ -71,7 +88,9 @@ Replace ```` with your JWT token. 6. Upgrade the Flyte Helm release. - helm upgrade flyte-core flyteorg/flyte-core -f https://raw.githubusercontent.com/flyteorg/flyte/master/charts/flyte-core/values-sandbox.yaml -f values-override.yaml -n flyte +.. code-block:: bash + + helm upgrade -n flyte -f values-override.yaml flyteorg/flyte-core 7. Register the Snowflake plugin example. @@ -100,4 +119,4 @@ Replace ```` with your JWT token. .. code-block:: bash - flytectl --config ~/.flyte/flytectl.yaml create execution -p -d --execFile ~/exec_spec.yaml + flytectl --config ~/.flyte/flytectl.yaml create execution -p -d --execFile ~/exec_spec.yaml \ No newline at end of file