fix(metadata-ingestion)glue connector failure when Optional field Type of PartitionKey is absent for a Table

metadata-ingestion/src/datahub/ingestion/source/aws/glue.py

@@ -1042,6 +1042,10 @@ def _transform_extraction(self) -> Iterable[MetadataWorkUnit]:
     def _extract_record(
         self, dataset_urn: str, table: Dict, table_name: str
     ) -> MetadataChangeEvent:
+        logger.debug(
+            f"extract record from table={table_name} for dataset={dataset_urn}"
+        )
+
         def get_owner() -> Optional[OwnershipClass]:
             owner = table.get("Owner")
             if owner:
@@ -1163,7 +1167,7 @@ def get_schema_metadata() -> Optional[SchemaMetadata]:
             for partition_key in partition_keys:
                 schema_fields = get_schema_fields_for_hive_column(
                     hive_column_name=partition_key["Name"],
-                    hive_column_type=partition_key["Type"],
+                    hive_column_type=partition_key.get("Type", "unknown"),
                     default_nullable=False,
                 )
                 assert schema_fields

metadata-ingestion/src/datahub/utilities/hive_schema_to_avro.py

@@ -28,6 +28,7 @@ class HiveColumnToAvroConverter:
         "bigint": "long",
         "varchar": "string",
         "char": "string",
+        "unknown": "null",
     }
 
     _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")

metadata-ingestion/tests/unit/utilities/test_hive_schema_to_avro.py

@@ -35,3 +35,11 @@ def test_get_avro_schema_for_struct_hive_with_duplicate_column2():
     assert schema_fields[0].type.type == NullTypeClass()
     assert schema_fields[0].fieldPath == "test"
     assert schema_fields[0].nativeDataType == invalid_schema
+
+
+def test_get_avro_schema_for_null_type_hive_column():
+    schema_fields = get_schema_fields_for_hive_column(
+        hive_column_name="test", hive_column_type="unknown"
+    )
+    assert schema_fields[0].type.type == NullTypeClass()
+    assert len(schema_fields) == 1