fix(ingest/databricks): handle and report config parse failure, updat…

metadata-ingestion/docs/sources/databricks/unity-catalog_pre.md

@@ -13,7 +13,7 @@
     * Ownership of or `SELECT` privilege on any tables and views you want to ingest
     * [Ownership documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/ownership.html)
     * [Privileges documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/privileges.html)
-  + To ingest legacy hive_metastore catalog (`include_hive_metastore` - disabled by default), your service principal must have all of the following:
+  + To ingest legacy hive_metastore catalog (`include_hive_metastore` - enabled by default), your service principal must have all of the following:
     * `READ_METADATA` and `USAGE` privilege on `hive_metastore` catalog
     * `READ_METADATA` and `USAGE` privilege on schemas you want to ingest
     * `READ_METADATA` and `USAGE` privilege on tables and views you want to ingest

metadata-ingestion/src/datahub/ingestion/source/unity/connection_test.py

@@ -11,8 +11,8 @@
 
 
 class UnityCatalogConnectionTest:
-    def __init__(self, config_dict: dict):
-        self.config = UnityCatalogSourceConfig.parse_obj_allow_extras(config_dict)
+    def __init__(self, config: UnityCatalogSourceConfig):
+        self.config = config
         self.report = UnityCatalogReport()
         self.proxy = UnityCatalogApiProxy(
             self.config.workspace_url,

metadata-ingestion/src/datahub/ingestion/source/unity/source.py

@@ -236,7 +236,14 @@ def init_hive_metastore_proxy(self):
 
     @staticmethod
     def test_connection(config_dict: dict) -> TestConnectionReport:
-        return UnityCatalogConnectionTest(config_dict).get_connection_test()
+        try:
+            config = UnityCatalogSourceConfig.parse_obj_allow_extras(config_dict)
+        except Exception as e:
+            return TestConnectionReport(
+                internal_failure=True,
+                internal_failure_reason=f"Failed to parse config due to {e}",
+            )
+        return UnityCatalogConnectionTest(config).get_connection_test()
 
     @classmethod
     def create(cls, config_dict, ctx):

metadata-ingestion/tests/unit/test_unity_catalog_config.py

@@ -4,6 +4,7 @@
 from freezegun import freeze_time
 
 from datahub.ingestion.source.unity.config import UnityCatalogSourceConfig
+from datahub.ingestion.source.unity.source import UnityCatalogSource
 
 FROZEN_TIME = datetime.fromisoformat("2023-01-01 00:00:00+00:00")
 
@@ -130,6 +131,17 @@ def test_warehouse_id_must_be_set_if_include_hive_metastore_is_true():
         )
 
 
+def test_warehouse_id_must_be_present_test_connection():
+    config_dict = {
+        "token": "token",
+        "workspace_url": "https://XXXXXXXXXXXXXXXXXXXXX",
+        "include_hive_metastore": True,
+    }
+    report = UnityCatalogSource.test_connection(config_dict)
+    assert report.internal_failure
+    print(report.internal_failure_reason)
+
+
 def test_set_profiling_warehouse_id_from_global():
     config = UnityCatalogSourceConfig.parse_obj(
         {