feat(graph): support using elasticsearch as graph backend.

datahub-kubernetes/datahub/README.md

@@ -57,6 +57,7 @@ helm install datahub datahub/
 | global.sql.datasource.username | string | `"root"` | SQL user name |
 | global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL password |
 | global.sql.datasource.password.secretKey | string | `"mysql-password"` | Secret key that contains the MySQL password |
+| global.graph_service_impl | string | `neo4j` | One of `neo4j` or `elasticsearch`. Determines which backend to use for the GMS graph service. Elastic is recommended for a simplified deployment. Neo4j will be the default for now to maintain backwards compatibility.
 
 ## Optional Chart Values
 

datahub-kubernetes/datahub/charts/datahub-gms/README.md

@@ -36,6 +36,7 @@ Current chart version is `0.2.0`
 | global.sql.datasource.username | string | `"datahub"` |  |
 | global.sql.datasource.password.secretRef | string | `"mysql-secrets"` |  |
 | global.sql.datasource.password.secretKey | string | `"mysql-password"` |  |
+| global.graph_service_impl | string | `neo4j` | One of `neo4j` or `elasticsearch`. Determines which backend to use for the GMS graph service. Elastic is recommended for a simplified deployment. Neo4j will be the default for now to maintain backwards compatibility.
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-gms"` |  |
 | image.tag | string | `"v0.8.3"` |  |

datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml

@@ -115,6 +115,9 @@ spec:
                   name: "{{ .password.secretRef }}"
                   key: "{{ .password.secretKey }}"
             {{- end }}
+            - name: GRAPH_SERVICE_IMPL
+              value: {{ .Values.global.graph_service_impl }}
+            {{- if eq .Values.global.graph_service_impl "neo4j" }}
             - name: NEO4J_HOST
               value: "{{ .Values.global.neo4j.host }}"
             - name: NEO4J_URI
@@ -126,6 +129,7 @@ spec:
                 secretKeyRef:
                   name: "{{ .Values.global.neo4j.password.secretRef }}"
                   key: "{{ .Values.global.neo4j.password.secretKey }}"
+            {{- end }}
             {{- if .Values.global.springKafkaConfigurationOverrides }}
             {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
             - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}

datahub-kubernetes/datahub/charts/datahub-gms/values.yaml

@@ -151,6 +151,7 @@ readinessProbe:
 # helm install datahub-gms datahub-gms/
 global:
   datahub_analytics_enabled: true
+  graph_service_impl: neo4j
 
   elasticsearch:
     host: "elasticsearch"
@@ -191,4 +192,4 @@ global:
         - "broker"
         - "mysql"
         - "elasticsearch"
-        - "neo4j"
+        - "neo4j"

datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md

@@ -29,6 +29,7 @@ Current chart version is `0.2.0`
 | global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` |  |
 | global.hostAliases[0].hostnames[3] | string | `"neo4j"` |  |
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
+| global.graph_service_impl | string | `neo4j` | One of `neo4j` or `elasticsearch`. Determines which backend to use for the GMS graph service. Elastic is recommended for a simplified deployment. Neo4j will be the default for now to maintain backwards compatibility.
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-mae-consumer"` |  |
 | image.tag | string | `"v0.8.3"` |  |

datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml

@@ -100,6 +100,9 @@ spec:
                   name: "{{ .password.secretRef }}"
                   key: "{{ .password.secretKey }}"
             {{- end }}
+            - name: GRAPH_SERVICE_IMPL
+              value: {{ .Values.global.graph_service_impl }}
+            {{- if eq .Values.global.graph_service_impl "neo4j" }}
             - name: NEO4J_HOST
               value: "{{ .Values.global.neo4j.host }}"
             - name: NEO4J_URI
@@ -111,6 +114,7 @@ spec:
                 secretKeyRef:
                   name: "{{ .Values.global.neo4j.password.secretRef }}"
                   key: "{{ .Values.global.neo4j.password.secretKey }}"
+            {{- end }}
             - name: DATAHUB_ANALYTICS_ENABLED
               value: "{{ .Values.global.datahub_analytics_enabled }}"
             {{- if .Values.global.springKafkaConfigurationOverrides }}

datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml

@@ -145,6 +145,7 @@ readinessProbe:
   failureThreshold: 8
 
 global:
+  graph_service_impl: neo4j
   datahub_analytics_enabled: true
 
   elasticsearch:
@@ -175,4 +176,4 @@ global:
         - "broker"
         - "mysql"
         - "elasticsearch"
-        - "neo4j"
+        - "neo4j"

datahub-kubernetes/datahub/quickstart-values-without-neo4j.yaml

@@ -0,0 +1,78 @@
+# Values to start up datahub after starting up the datahub-prerequisites chart with "prerequisites" release name
+# Copy this chart and change configuration as needed.
+datahub-gms:
+  enabled: true
+  image:
+    repository: linkedin/datahub-gms
+    tag: "v0.8.1"
+
+datahub-frontend:
+  enabled: true
+  image:
+    repository: linkedin/datahub-frontend-react
+    tag: "v0.8.1"
+  # Set up ingress to expose react front-end
+  ingress:
+    enabled: false
+
+elasticsearchSetupJob:
+  enabled: true
+  image:
+    repository: linkedin/datahub-elasticsearch-setup
+    tag: "v0.8.1"
+
+kafkaSetupJob:
+  enabled: true
+  image:
+    repository: linkedin/datahub-kafka-setup
+    tag: "v0.8.1"
+
+mysqlSetupJob:
+  enabled: true
+  image:
+    repository: acryldata/datahub-mysql-setup
+    tag: "v0.8.1"
+
+datahubUpgrade:
+  enabled: true
+  image:
+    repository: acryldata/datahub-upgrade
+    tag: "v0.8.1"
+
+datahub-ingestion-cron:
+  enabled: false
+
+global:
+  graph_service_impl: elasticsearch
+
+  elasticsearch:
+    host: "elasticsearch-master"
+    port: "9200"
+    indexPrefix: demo
+
+  kafka:
+    bootstrap:
+      server: "prerequisites-kafka:9092"
+    zookeeper:
+      server: "prerequisites-zookeeper:2181"
+    schemaregistry:
+      url: "http://prerequisites-cp-schema-registry:8081"
+
+  sql:
+    datasource:
+      host: "prerequisites-mysql:3306"
+      hostForMysqlClient: "prerequisites-mysql"
+      port: "3306"
+      url: "jdbc:mysql://prerequisites-mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8"
+      driver: "com.mysql.jdbc.Driver"
+      username: "root"
+      password:
+        secretRef: mysql-secrets
+        secretKey: mysql-root-password
+
+  datahub:
+    gms:
+      port: "8080"
+    mae_consumer:
+      port: "9091"
+    appVersion: "1.0"

datahub-kubernetes/datahub/quickstart-values.yaml

@@ -43,6 +43,8 @@ datahub-ingestion-cron:
   enabled: false
 
 global:
+  graph_service_impl: neo4j
+
   elasticsearch:
     host: "elasticsearch-master"
     port: "9200"

datahub-kubernetes/datahub/values.yaml

@@ -58,7 +58,7 @@ datahubUpgrade:
     tag: "v0.8.3"
 
 global:
-
+  graph_service_impl: neo4j
   datahub_analytics_enabled: true
   datahub_standalone_consumers_enabled: false
 

docker/datahub-gms/env/docker-without-neo4j.env

@@ -0,0 +1,39 @@
+DATASET_ENABLE_SCSI=false
+EBEAN_DATASOURCE_USERNAME=datahub
+EBEAN_DATASOURCE_PASSWORD=datahub
+EBEAN_DATASOURCE_HOST=mysql:3306
+EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
+EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
+KAFKA_BOOTSTRAP_SERVER=broker:29092
+KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+ELASTICSEARCH_HOST=elasticsearch
+ELASTICSEARCH_PORT=9200
+GRAPH_SERVICE_IMPL=elasticsearch
+
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true
+
+# Uncomment to disable persistence of client-side analytics events
+# DATAHUB_ANALYTICS_ENABLED=false
+
+# Uncomment to configure kafka topic names
+# Make sure these names are consistent across the whole deployment
+# METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
+# METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4
+# FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
+
+# Uncomment and set these to support SSL connection to Elasticsearch
+# ELASTICSEARCH_USE_SSL=true
+# ELASTICSEARCH_SSL_PROTOCOL=TLSv1.2
+# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL=
+# ELASTICSEARCH_SSL_TRUSTSTORE_FILE=
+# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE=
+# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD=
+# ELASTICSEARCH_SSL_KEYSTORE_FILE=
+# ELASTICSEARCH_SSL_KEYSTORE_TYPE=
+# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD=
+
+# To use simple username/password authentication to Elasticsearch over HTTPS
+# set ELASTICSEARCH_USE_SSL=true and uncomment:
+# ELASTICSEARCH_USERNAME=
+# ELASTICSEARCH_PASSWORD=

docker/datahub-gms/env/docker.env

@@ -12,6 +12,7 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+GRAPH_SERVICE_IMPL=neo4j
 
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true

docker/datahub-gms/start.sh

@@ -30,7 +30,6 @@ dockerize \
   -wait tcp://$EBEAN_DATASOURCE_HOST \
   -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
   -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
-  -wait $NEO4J_HOST \
   -timeout 240s \
   java $JAVA_OPTS $JMX_OPTS \
   -jar /jetty-runner.jar \

docker/docker-compose-without-neo4j.override.yml

@@ -0,0 +1,33 @@
+---
+version: '3.8'
+services:
+  mysql:
+    container_name: mysql
+    hostname: mysql
+    image: mysql:5.7
+    env_file: mysql/env/docker.env
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    ports:
+      - "3306:3306"
+    volumes:
+      - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+      - mysqldata:/var/lib/mysql
+
+  mysql-setup:
+    build:
+      context: ../
+      dockerfile: docker/mysql-setup/Dockerfile
+    image: acryldata/datahub-mysql-setup:head
+    env_file: mysql-setup/env/docker.env
+    hostname: mysql-setup
+    container_name: mysql-setup
+    depends_on:
+      - mysql
+
+  datahub-gms:
+    env_file: datahub-gms/env/docker-without-neo4j.env
+    depends_on:
+      - mysql
+
+volumes:
+  mysqldata: