feat(build): changes to decrease build time, cancel runs in case of multiple commits

.github/workflows/build-and-test.yml

@@ -16,6 +16,10 @@ on:
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/check-datahub-jars.yml

@@ -17,6 +17,10 @@ on:
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   check_jars:

.github/workflows/docker-unified.yml

@@ -15,6 +15,10 @@ on:
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DATAHUB_GMS_IMAGE: 'linkedin/datahub-gms'
   DATAHUB_FRONTEND_IMAGE: 'linkedin/datahub-frontend-react'

.github/workflows/documentation.yml

@@ -10,6 +10,10 @@ on:
   # release:
   #   types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   gh-pages:
     runs-on: ubuntu-latest

.github/workflows/metadata-ingestion.yml

@@ -15,12 +15,17 @@ on:
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   metadata-ingestion-general:
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 3.0.3
+      DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
         python-version: ["3.6", "3.9"]
@@ -29,30 +34,28 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
-        with:
-          spark-version: '3.0.3'
-          hadoop-version: '3.2'
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run metadata-ingestion tests
-        run: ./gradlew :metadata-ingestion:build :metadata-ingestion:check
+        run: ./gradlew :metadata-ingestion:build :metadata-ingestion:testQuick :metadata-ingestion:check
       - uses: actions/upload-artifact@v2
         if: always()
         with:
-          name: Test Results (metadata ingestion general)
+          name: Test Results (metadata ingestion ${{ matrix.python-version }} testQuick)
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
 
-  metadata-ingestion-by-version:
+  metadata-ingestion:
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 3.0.3
+      DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
         python-version: ["3.6", "3.9"]
+        command: ["installAirflow1", "testIntegration", "testSlowIntegration"]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -64,11 +67,9 @@ jobs:
           spark-version: '3.0.3'
           hadoop-version: '3.2'
       - name: Install dependencies
-        run: ./metadata-ingestion/scripts/install_deps.sh && python -m pip install --upgrade pip && pip install tox tox-gh-actions
-      - name: Codegen
-        run: ./gradlew :metadata-ingestion:codegen
-      - name: Run tox tests
-        run: cd metadata-ingestion && tox
+        run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Run metadata-ingestion tests
+        run: ./gradlew :metadata-ingestion:build :metadata-ingestion:${{ matrix.command }} -x:metadata-ingestion:testQuick -x:metadata-ingestion:check
       - uses: actions/upload-artifact@v2
         if: always()
         with:

metadata-ingestion/build.gradle

@@ -84,10 +84,10 @@ task testQuick(type: Exec, dependsOn: installDev) {
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
   commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -m 'not integration and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "source ${venv_name}/bin/activate && pytest --durations=20 -m 'not slow_unit and not integration and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
-task installDevTest(type: Exec, dependsOn: [installDev]) {
+task installDevTest(type: Exec, dependsOn: [install]) {
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
   outputs.file("${venv_name}/.build_install_dev_test_sentinel")
@@ -109,14 +109,27 @@ task testSingle(dependsOn: [installDevTest]) {
   }
 }
 
-task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
+task installAirflow1(type: Exec, dependsOn: [install]) {
+  inputs.file file('setup.py')
+  outputs.dir("${venv_name}")
+  outputs.file("${venv_name}/.build_install_airflow_sentinel")
+  commandLine 'bash', '-x', '-c',
+    "${venv_name}/bin/pip install -e .[dev-airflow1] -c tests/airflow1-constraints.txt && touch ${venv_name}/.build_install_airflow_sentinel"
+}
+
+task testIntegration(type: Exec, dependsOn: [installDevTest]) {
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && pytest --durations=50 -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml"
+}
+
+task testFull(type: Exec, dependsOn: [installDevTest]) {
   commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+    "source ${venv_name}/bin/activate && pytest --durations=50 -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
 
-task testSlowIntegration(type: Exec, dependsOn: [testQuick, installDevTest]) {
+task testSlowIntegration(type: Exec, dependsOn: [installDevTest]) {
   commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -m 'slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+    "source ${venv_name}/bin/activate && pytest --durations=20 -m 'slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.slow.integration.xml"
 }
 
 task docGen(type: Exec, dependsOn: [codegen, installDevTest]) {

metadata-ingestion/setup.cfg

@@ -54,9 +54,9 @@ disallow_untyped_defs = yes
 asyncio_mode = auto
 addopts = --cov=src --cov-report term-missing --cov-config setup.cfg --strict-markers
 markers =
+    slow_unit: marks tests to only run slow unit tests (deselect with '-m not slow_unit')
     integration: marks tests to only run in integration (deselect with '-m "not integration"')
     slow_integration: marks tests that are too slow to even run in integration (deselect with '-m "not slow_integration"')
-    slow_hana: marks tests that are too slow to even run in integration (deselect with '-m "not slow_hana"')
 testpaths = 
     tests/unit
     tests/integration

metadata-ingestion/setup.py

@@ -321,7 +321,6 @@ def get_long_description():
     "pytest-asyncio>=0.16.0",
     "pytest-cov>=2.8.1",
     "pytest-docker>=0.10.3,<0.12",
-    "tox",
     "deepdiff",
     "requests-mock",
     "freezegun",

metadata-ingestion/src/datahub/ingestion/graph/client.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from json.decoder import JSONDecodeError
 from typing import Any, Dict, List, Optional, Type
 
@@ -24,6 +25,11 @@
 logger = logging.getLogger(__name__)
 
 
+telemetry_enabled = (
+    os.environ.get("DATAHUB_TELEMETRY_ENABLED", "true").lower() == "true"
+)
+
+
 class DatahubClientConfig(ConfigModel):
     """Configuration class for holding connectivity to datahub gms"""
 
@@ -51,6 +57,9 @@ def __init__(self, config: DatahubClientConfig) -> None:
             ca_certificate_path=self.config.ca_certificate_path,
         )
         self.test_connection()
+        if not telemetry_enabled:
+            self.server_id = "missing"
+            return
         try:
             client_id: Optional[TelemetryClientIdClass] = self.get_aspect_v2(
                 "urn:li:telemetry:clientId", TelemetryClientIdClass, "telemetryClientId"

metadata-ingestion/tests/integration/s3/test_s3.py

@@ -69,6 +69,7 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_name):
 source_files = os.listdir(SOURCE_FILES_PATH)
 
 
+@pytest.mark.slow_unit
 @pytest.mark.parametrize("source_file", source_files)
 def test_data_lake_s3_ingest(
     pytestconfig, s3_populate, source_file, tmp_path, mock_time
@@ -101,6 +102,7 @@ def test_data_lake_s3_ingest(
     )
 
 
+@pytest.mark.slow_unit
 @pytest.mark.parametrize("source_file", source_files)
 def test_data_lake_local_ingest(pytestconfig, source_file, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/s3/"

metadata-ingestion/tests/integration/tableau/test_tableau.py

@@ -2,6 +2,7 @@
 import pathlib
 from unittest import mock
 
+import pytest
 from freezegun import freeze_time
 
 from datahub.ingestion.run.pipeline import Pipeline
@@ -40,6 +41,7 @@ def side_effect_query_metadata(query):
 
 
 @freeze_time(FROZEN_TIME)
+@pytest.mark.slow_unit
 def test_tableau_ingest(pytestconfig, tmp_path):
 
     global test_resources_dir