feat: No Code Metadata Modeling (#2629)

Co-authored-by: Dexter Lee <[email protected]> Co-authored-by: Gabe Lyons <[email protected]> Co-authored-by: Shirshanka Das <[email protected]>
datahub-project · Jun 3, 2021 · 97e9660 · 97e9660
1 parent 7a0bab6
commit 97e9660
Show file tree

Hide file tree

Showing 405 changed files with 25,185 additions and 1,758 deletions.
diff --git a/.github/workflows/docker-datahub-upgrade.yml b/.github/workflows/docker-datahub-upgrade.yml
@@ -0,0 +1,74 @@
+name: docker-datahub-upgrade docker
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - './datahub-upgrade/**'
+      - '.github/workflows/docker-datahub-upgrade.yml'
+    paths_ignore:
+      - '**.md'
+      - '**.env'
+  release:
+    types: [published, edited]
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+      publish: ${{ steps.publish.outputs.publish }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Compute Tag
+        id: tag
+        run: |
+          echo "GITHUB_REF: $GITHUB_REF"
+          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
+          echo "SHORT_SHA: $SHORT_SHA"
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/.*$,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          echo "tag=$TAG"
+          echo "::set-output name=tag::$TAG"
+      - name: Check whether publishing enabled
+        id: publish
+        env:
+          ENABLE_PUBLISH: ${{ secrets.ORG_DOCKER_PASSWORD }}
+        run: |
+          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
+          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+  push_to_registries:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    if: ${{ needs.setup.outputs.publish == 'true' }}
+    needs: setup
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            acryldata/datahub-upgrade
+          # add git short SHA as Docker tag
+          tag-custom: ${{ needs.setup.outputs.tag }}
+          tag-custom-only: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.ORG_DOCKER_PASSWORD }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./docker/datahub-upgrade/Dockerfile
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-elasticsearch-setup.yml b/.github/workflows/docker-elasticsearch-setup.yml
@@ -30,7 +30,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -68,4 +68,4 @@ jobs:
         with:
           file: ./docker/elasticsearch-setup/Dockerfile
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-frontend-react.yml b/.github/workflows/docker-frontend-react.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled

diff --git a/.github/workflows/docker-gms.yml b/.github/workflows/docker-gms.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -70,4 +70,4 @@ jobs:
         with:
           file: ./docker/datahub-gms/Dockerfile
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled

diff --git a/.github/workflows/docker-kafka-setup.yml b/.github/workflows/docker-kafka-setup.yml
@@ -29,7 +29,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -68,4 +68,4 @@ jobs:
           file: ./docker/kafka-setup/Dockerfile
           context: ./docker/kafka-setup
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled

diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled

diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,9 @@
 *.iws
 *.ipr
 
+*gma-create-all.sql
+*gma-drop-all.sql
+
 # Pegasus & Avro
 **/src/mainGenerated*
 **/src/testGenerated*

diff --git a/build.gradle b/build.gradle
@@ -9,7 +9,7 @@ buildscript {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
-    classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.2.RELEASE'
+    classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.4.RELEASE'
   }
 }
 
@@ -38,6 +38,7 @@ project.ext.externalDependency = [
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
+    'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
     'ebean': 'io.ebean:ebean:11.33.3',
     'ebeanAgent': 'io.ebean:ebean-agent:11.27.1',
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
@@ -62,6 +63,7 @@ project.ext.externalDependency = [
     'h2': 'com.h2database:h2:1.4.196',
     'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
+    'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
     'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.9.7',
     'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.9.7',
     'javatuples': 'org.javatuples:javatuples:1.2',
@@ -79,6 +81,7 @@ project.ext.externalDependency = [
     'lombok': 'org.projectlombok:lombok:1.18.12',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mockito': 'org.mockito:mockito-core:3.0.0',
+    'mockitoInline': 'org.mockito:mockito-inline:3.0.0',
     'mysqlConnector': 'mysql:mysql-connector-java:5.1.47',
     'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
     'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
@@ -101,12 +104,15 @@ project.ext.externalDependency = [
     'springCore': 'org.springframework:spring-core:5.2.3.RELEASE',
     'springJdbc': 'org.springframework:spring-jdbc:5.2.3.RELEASE',
     'springWeb': 'org.springframework:spring-web:5.2.3.RELEASE',
-    'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.2.RELEASE',
-    'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.2.RELEASE',
-    'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.2.RELEASE',
+    'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.4.RELEASE',
+    'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.4.RELEASE',
+    'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.4.RELEASE',
     'springKafka': 'org.springframework.kafka:spring-kafka:2.2.14.RELEASE',
     'springActuator': 'org.springframework.boot:spring-boot-starter-actuator:2.1.2.RELEASE',
-    'testng': 'org.testng:testng:6.9.9',
+    'testng': 'org.testng:testng:7.3.0',
+    'testContainers': 'org.testcontainers:testcontainers:1.15.1',
+    'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.1',
+    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:1.15.3',
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14'
 ]
 

diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle
@@ -42,6 +42,7 @@ dependencies {
   playRun externalDependency.guice
   playRun externalDependency.playDocs
   playRun externalDependency.playGuice
+  playRun externalDependency.logbackClassic
 }
 
 model {

diff --git a/datahub-frontend/run/frontend.env b/datahub-frontend/run/frontend.env
@@ -22,9 +22,9 @@ DATAHUB_GMS_PORT=8080
 # DATAHUB_ANALYTICS_ENABLED=false
 
 # Kafka Producer Configuration
-KAFKA_BOOTSTRAP_SERVER=localhost:29092
+KAFKA_BOOTSTRAP_SERVER=localhost:9092
 DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
 
 # Required Elastic Client Configuration (Analytics)
 ELASTIC_CLIENT_HOST=localhost
-ELASTIC_CLIENT_PORT=9200
+ELASTIC_CLIENT_PORT=9200
diff --git a/datahub-frontend/run/logback.xml b/datahub-frontend/run/logback.xml
@@ -27,8 +27,8 @@
   <logger name="com.linkedin.datahub" level="DEBUG" />
   <logger name="play.core.server.AkkaHttpServer" level="INFO" />
 
-  <root level="warn">
+  <root level="info">
     <appender-ref ref="STDOUT"/>
     <appender-ref ref="FILE"/>
   </root>
-</configuration>
+</configuration>
diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle
@@ -11,6 +11,7 @@ dependencies {
     compile externalDependency.antlr4Runtime
     compile externalDependency.antlr4
     compile externalDependency.guava
+
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok