build(ingest): use gradle in commands + docs (#2531)

datahub-project · May 12, 2021 · a47400f · a47400f
1 parent a2755cd
commit a47400f
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 18 deletions.
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -29,14 +29,21 @@ task installDev(type: Exec, dependsOn: [environmentSetup, codegen]) {
   commandLine "${venv_name}/bin/pip", 'install', '-e', '.[dev]'
 }
 task lint(type: Exec, dependsOn: installDev) {
-  commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && black --check src/ tests/ && isort --check-only src/ tests/ && flake8 --count --statistics src/ tests/ && mypy src/ tests/"
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && black --check src/ tests/ && isort --check-only src/ tests/ && flake8 --count --statistics src/ tests/ && mypy src/ tests/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {
-  commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && black src/ tests/ && isort src/ tests/ && flake8 src/ tests/ && mypy src/ tests/"
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && " +
+    "black src/ tests/ && " +
+    "isort src/ tests/ && " +
+    "flake8 src/ tests/ && " +
+    "mypy src/ tests/"
 }
 
 task test(type: Exec, dependsOn: installDev) {
-  commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest -vv"
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && pytest -vv"
 }
 
 build.dependsOn install

diff --git a/metadata-ingestion/developing.md b/metadata-ingestion/developing.md
@@ -13,24 +13,16 @@ The architecture of this metadata ingestion framework is heavily inspired by [Ap
 ### Requirements
 
 1. Python 3.6+ must be installed in your host environment.
-2. You also need to build the `mxe-schemas` module as below.
-   ```
-   (cd .. && ./gradlew :metadata-events:mxe-schemas:build)
-   ```
-   This is needed to generate `MetadataChangeEvent.avsc` which is the schema for the `MetadataChangeEvent_v4` Kafka topic.
-3. On MacOS: `brew install librdkafka`
-4. On Debian/Ubuntu: `sudo apt install librdkafka-dev python3-dev python3-venv`
-5. On Fedora (if using LDAP source integration): `sudo yum install openldap-devel`
+2. On MacOS: `brew install librdkafka`
+3. On Debian/Ubuntu: `sudo apt install librdkafka-dev python3-dev python3-venv`
+4. On Fedora (if using LDAP source integration): `sudo yum install openldap-devel`
 
 ### Set up your Python environment
 
 ```shell
-python3 -m venv venv
+../gradlew :metadata-ingestion:installDev
 source venv/bin/activate
-pip install --upgrade pip wheel setuptools
-pip uninstall datahub || true ; rm -r src/*.egg-info || true
-pip install -e .
-(cd .. && ./gradlew :metadata-events:mxe-schemas:build) && ./scripts/codegen.sh
+datahub version  # check that it works
 ```
 
 ### Common setup issues
@@ -115,4 +107,7 @@ isort .
 flake8 .
 mypy .
 pytest
+
+# These steps are all included in the gradle build:
+../gradlew :metadata-ingestion:check
 ```
diff --git a/metadata-ingestion/scripts/release.sh b/metadata-ingestion/scripts/release.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
-./gradlew build  # also runs tests
+../gradlew build  # also runs tests
 
 rm -rf build dist || true
 python -m build

diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/test_pipeline.py
@@ -31,6 +31,7 @@ def test_configure(self, mock_sink, mock_source):
         )
         pipeline.run()
         pipeline.raise_from_status()
+        pipeline.pretty_print_summary()
         mock_source.assert_called_once()
         mock_sink.assert_called_once()
 

diff --git a/metadata-ingestion/tox.ini b/metadata-ingestion/tox.ini
@@ -8,7 +8,7 @@ envlist = py3-airflow{2,1}
 
 [testenv]
 setenv =
-    AIRFLOW_HOME = "/tmp/airflow/thisshouldnotexist-{envname}"
+    AIRFLOW_HOME = /tmp/airflow/thisshouldnotexist-{envname}
 extras = dev
 commands =
     # TODO: look at https://github.com/pytest-dev/pytest-cov/blob/master/examples/src-layout/tox.ini