From 1b9920f98160514df026656f73675a21d58d2241 Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Wed, 18 Jan 2023 17:30:47 +0200 Subject: [PATCH] feat: add the ElasticSearch helm chart for cluster-level ES --- .gitignore | 1 + README.md | 26 ++++++- infra-example/k8s-cluster/main.tf | 6 +- .../tutor_multi_k8s_plugin/commands.py | 44 ++++++++++++ .../tutor_multi_k8s_plugin/elasticsearch.py | 70 +++++++++++++++++++ .../patches/k8s-services | 4 +- .../patches/openedx-common-settings | 10 +++ ...penedx-dockerfile-post-python-requirements | 7 ++ .../tutor_multi_k8s_plugin/plugin.py | 19 +++-- tutor-multi-chart/Chart.lock | 7 +- tutor-multi-chart/Chart.yaml | 4 ++ .../templates/elasticsearch/secrets.yaml | 20 ++++++ tutor-multi-chart/values.yaml | 48 +++++++++++++ values-example.yaml | 3 + 14 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/commands.py create mode 100644 tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/elasticsearch.py create mode 100644 tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-common-settings create mode 100644 tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-dockerfile-post-python-requirements create mode 100644 tutor-multi-chart/templates/elasticsearch/secrets.yaml diff --git a/.gitignore b/.gitignore index 261a82e..430af72 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ infra-*/.terraform* infra-*/secrets.auto.tfvars my-notes values.yaml +tutor-multi-chart/charts/*.tgz diff --git a/README.md b/README.md index 475adda..0d62013 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,31 @@ of the box. ---------------- -## Appendix A: how to uninstall this chart +## Configuration + +### Multi-tenant Elasticsearch + +Tutor creates an Elasticsearch pod as part of the Kubernetes deployment. Depending on the number of instances +Memory and CPU use can be lowered by running a central ES cluster instead of an ES pod for every instance. + +To enable set `elasticsearch.enabled=true` in your `values.yaml` and deploy the chart. + +For each instance you would like to enable this on, set the configuration values in the respective `config.yml`: + +```yaml +K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH: true +RUN_ELASTICSEARCH: false +``` + +- And create the user on the cluster with `tutor k8s harmony create-elasticsearch-user`. +- Rebuild your Open edX image `tutor images build openedx`. +- Finally, redeploy your changes: `tutor k8s start && tutor k8s init`. + +#### Caveats + +In order for SSL to work without warnings the CA certificate needs to be mounted in the relevant pods. This is not yet implemented as due to an [outstanding issue in tutor](https://github.com/overhangio/tutor/issues/791) that had not yet been completed at the time of writing. + +## Appendix : how to uninstall this chart Just run `helm uninstall --namespace tutor-multi tutor-multi` to uninstall this. diff --git a/infra-example/k8s-cluster/main.tf b/infra-example/k8s-cluster/main.tf index fb31cf7..5bd2c5f 100644 --- a/infra-example/k8s-cluster/main.tf +++ b/infra-example/k8s-cluster/main.tf @@ -26,10 +26,10 @@ resource "digitalocean_kubernetes_cluster" "cluster" { node_pool { name = "${var.cluster_name}-nodes" - size = "s-2vcpu-4gb" - # At this size, at least 4 nodes are recommended to run 2 Open edX instances using the default Tutor images, because + size = "s-4vcpu-8gb" + # At this size, at least 3 nodes are recommended to run 2 Open edX instances using the default Tutor images, because # resources like MySQL/MongoDB are not shared. - min_nodes = 4 + min_nodes = 3 max_nodes = 4 auto_scale = true } diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/commands.py b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/commands.py new file mode 100644 index 0000000..643d019 --- /dev/null +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/commands.py @@ -0,0 +1,44 @@ +import os + +import click +from tutor import config as tutor_config +from tutor import env as tutor_env +from tutor.commands.k8s import K8sContext, kubectl_exec +from .elasticsearch import ElasticSearchAPI + +@click.group(help="Commands and subcommands of the openedx-k8s-harmony.") +@click.pass_context +def harmony(context: click.Context) -> None: + context.obj = K8sContext(context.obj.root) + + +@click.command(help="Create or update Elasticsearch users") +@click.pass_obj +def create_elasticsearch_user(context: click.Context): + """ + Creates or updates the Elasticsearch user + """ + config = tutor_config.load(context.root) + namespace = config["K8S_HARMONY_NAMESPACE"] + api = ElasticSearchAPI(namespace) + username, password = config["ELASTICSEARCH_HTTP_AUTH"].split(":", 1) + role_name = f"{username}_role" + + prefix = config["ELASTICSEARCH_INDEX_PREFIX"] + api.post( + f"_security/role/{role_name}", + {"indices": [{"names": [f"{prefix}*"], "privileges": ["all"]}]}, + ) + + api.post( + f"_security/user/{username}", + { + "password": password, + "enabled": True, + "roles": [role_name], + "full_name": username, + }, + ) + + +harmony.add_command(create_elasticsearch_user) diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/elasticsearch.py b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/elasticsearch.py new file mode 100644 index 0000000..c654e31 --- /dev/null +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/elasticsearch.py @@ -0,0 +1,70 @@ +import json +import typing + +from tutor import utils + + +class ElasticSearchAPI: + """ + Helper class to interact with the ElasticSearch + API on the deployed cluster. + """ + + def __init__(self, namespace): + self._command_base = [ + "kubectl", + "exec", + "--stdin", + "--tty", + "--namespace", + namespace, + "elasticsearch-master-0", + "--", + "bash", + "-c", + ] + self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"] + + def run_command(self, curl_options) -> typing.Union[dict, bytes]: + """ + Invokes a curl command on the first Elasticsearch pod. + + If possible returns the parsed json from the Elasticsearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + response = utils.check_output( + *self._command_base, " ".join(self._curl_base + curl_options) + ) + try: + return json.loads(response) + except (TypeError, ValueError): + return response + + def get(self, endpoint): + """ + Runs a GET request on the Elasticsearch cluster with the specified + endpoint. + + If possible returns the parsed json from the Elasticsearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + return self.run_command(["-XGET", f"https://localhost:9200/{endpoint}"]) + + def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: + """ + Runs a POST request on the Elasticsearch cluster with the specified + endpoint. + + If possible returns the parsed json from the Elasticsearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + return self.run_command( + [ + "-XPOST", + f"https://localhost:9200/{endpoint}", + "-d", + f"'{json.dumps(data)}'", + "-H", + '"Content-Type: application/json"', + ] + ) diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/k8s-services b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/k8s-services index f09e976..8b44f6e 100644 --- a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/k8s-services +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/k8s-services @@ -2,7 +2,7 @@ # For multi-instance clusters. Allow one central load balancer on the cluster to # handle HTTPS certs and forward traffic to each Open edX instance's Caddy # instance. -{%- set HOSTS = [LMS_HOST, CMS_HOST, PREVIEW_HOST, MFE_HOST] + MULTI_K8S_INGRESS_HOST_LIST %} +{%- set HOSTS = [LMS_HOST, CMS_HOST, PREVIEW_HOST, MFE_HOST] + K8S_HARMONY_INGRESS_HOST_LIST %} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -12,7 +12,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: tutor-multi-letsencrypt-global spec: - ingressClassName: {{ MULTI_K8S_INGRESS_CLASS_NAME }} + ingressClassName: {{ K8S_HARMONY_INGRESS_CLASS_NAME }} rules: {%- for host in HOSTS if host is defined %} - host: "{{ host }}" diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-common-settings b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-common-settings new file mode 100644 index 0000000..c2e5611 --- /dev/null +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-common-settings @@ -0,0 +1,10 @@ +{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} +ELASTICSEARCH_INDEX_PREFIX = "{{ELASTICSEARCH_INDEX_PREFIX}}" +ELASTIC_SEARCH_CONFIG = [{ + "use_ssl": True, + "host": "elasticsearch-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local", + "verify_certs": False, + "port": 9200, + "http_auth": "{{ ELASTICSEARCH_HTTP_AUTH }}" +}] +{% endif %} diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-dockerfile-post-python-requirements b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-dockerfile-post-python-requirements new file mode 100644 index 0000000..660080a --- /dev/null +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-dockerfile-post-python-requirements @@ -0,0 +1,7 @@ +{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} +# This is needed otherwise the previously installed edx-search +# package doesn't get replaced. Once the below branch is merged +# upstream it will no longer be needed. +RUN pip uninstall -y edx-search +RUN pip install --upgrade git+https://github.com/open-craft/edx-search.git@keith/prefixed-index-names +{% endif %} diff --git a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/plugin.py b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/plugin.py index 7aafda9..fac5542 100644 --- a/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/plugin.py +++ b/tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/plugin.py @@ -3,6 +3,7 @@ import pkg_resources from tutor import hooks +from . import commands from .__about__ import __version__ @@ -19,7 +20,8 @@ # when installing additional plugins such as tutor-ecommerce or tutor-minio. # The workaround is to manually add a list of hosts to be routed to the caddy # instance. - "INGRESS_HOST_LIST": [ ] + "INGRESS_HOST_LIST": [], + "ENABLE_SHARED_ELASTICSEARCH": False, }, "overrides": { # Don't use Caddy as a per-instance external web proxy, but do still use it @@ -28,16 +30,19 @@ # We are using HTTPS "ENABLE_HTTPS": True, }, + "unique": { + "ELASTICSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}", + "ELASTICSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-", + }, } +# Load all configuration entries hooks.Filters.CONFIG_DEFAULTS.add_items( - [ - (f"MULTI_K8S_{key}", value) - for key, value in config.get("defaults", {}).items() - ] + [(f"K8S_HARMONY_{key}", value) for key, value in config["defaults"].items()] ) -hooks.Filters.CONFIG_OVERRIDES.add_items(list(config.get("overrides", {}).items())) +hooks.Filters.CONFIG_OVERRIDES.add_items(list(config["overrides"].items())) +hooks.Filters.CONFIG_UNIQUE.add_items(list(config["unique"].items())) # Load all patches from the "patches" folder for path in glob( @@ -48,3 +53,5 @@ ): with open(path, encoding="utf-8") as patch_file: hooks.Filters.ENV_PATCHES.add_item((os.path.basename(path), patch_file.read())) + +hooks.Filters.CLI_COMMANDS.add_item(commands.harmony) diff --git a/tutor-multi-chart/Chart.lock b/tutor-multi-chart/Chart.lock index ca39b25..ac4574d 100644 --- a/tutor-multi-chart/Chart.lock +++ b/tutor-multi-chart/Chart.lock @@ -5,5 +5,8 @@ dependencies: - name: cert-manager repository: https://charts.jetstack.io version: v1.11.0 -digest: sha256:7e80a648180d0fe471a9f45d2a47edd5da8894ef544b562911fac2e11d7fd411 -generated: "2023-02-02T14:28:19.400979073-04:00" +- name: elasticsearch + repository: https://helm.elastic.co + version: 7.17.3 +digest: sha256:833041ca860a77cc220bdc97f7b6af8ff8b6da0c0a021615dc2858138a29bbbd +generated: "2023-02-26T12:17:23.507774503+02:00" diff --git a/tutor-multi-chart/Chart.yaml b/tutor-multi-chart/Chart.yaml index 6576804..e29029b 100644 --- a/tutor-multi-chart/Chart.yaml +++ b/tutor-multi-chart/Chart.yaml @@ -25,3 +25,7 @@ dependencies: version: "1.11.0" repository: https://charts.jetstack.io condition: cert-manager.enabled + +- name: elasticsearch + version: "7.17.3" + repository: https://helm.elastic.co diff --git a/tutor-multi-chart/templates/elasticsearch/secrets.yaml b/tutor-multi-chart/templates/elasticsearch/secrets.yaml new file mode 100644 index 0000000..e25eecf --- /dev/null +++ b/tutor-multi-chart/templates/elasticsearch/secrets.yaml @@ -0,0 +1,20 @@ +--- +{{- $ca := genCA "elasticca" 1825 }} +{{- $cert := genSignedCert "elasticsearch-master.{{ Release.Namespace }}.local" nil (list "elasticsearch-master.{{ Release.Namespace }}.local") 1825 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-certificates +type: Opaque +data: + "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}} + "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}} + "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-credentials +type: Opaque +data: + "password": {{ randAlphaNum 32 | b64enc | quote }} diff --git a/tutor-multi-chart/values.yaml b/tutor-multi-chart/values.yaml index d61faf6..a5e4516 100644 --- a/tutor-multi-chart/values.yaml +++ b/tutor-multi-chart/values.yaml @@ -13,3 +13,51 @@ cert-manager: # Email address associated with the ACME account. Used to notify about expiring # certificates. email: "" + +# Multi-tenant ElasticSearch +elasticsearch: + enabled: false + + # Operators will need to add/update the following setting in each + # of their instances by running the commands: + # ``` + # tutor config save --set K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH=true --set RUN_ELASTICSEARCH=false + # tutor harmony create-elasticsearch-user + # ``` + # RUN_ELASTICSEARCH: false + # ELASTICSEARCH_PREFIX_INDEX: "username-" + # K8S_HARMONY_USE_SHARED_ELASTICSEARCH: true + # ELASTICSEARCH_AUTH: "username:actual_password" + + # We will create the relevant certs, because they need to shared + # with pods in other namespaces. + createCert: false + # Authentication is only available in https + protocol: https + + # This secret will contain the http certificates. + secretMounts: + - name: elasticsearch-certificates + secretName: elasticsearch-certificates + path: /usr/share/elasticsearch/config/certs + defaultMode: 0777 + + # The password for the elastic user is stored in this secret + extraEnvs: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-credentials + key: password + + esConfig: + "elasticsearch.yml": | + xpack.security.enabled: true + xpack.security.http.ssl.enabled: true + xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/tls.key + xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt + xpack.security.transport.ssl.enabled: true + xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/tls.key + xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt + xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca.crt + xpack.security.transport.ssl.verification_mode: certificate diff --git a/values-example.yaml b/values-example.yaml index 824c467..6f78498 100644 --- a/values-example.yaml +++ b/values-example.yaml @@ -5,3 +5,6 @@ ingress-nginx: cert-manager: # Set your email address here so auto-generated HTTPS certs will work: email: "email@example.com" + +elasticsearch: + enabled: false