Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ingest/lookml): ingest field tags #10792

Merged
merged 6 commits into from
Jul 1, 2024
Merged

feat(ingest/lookml): ingest field tags #10792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
27dbba4
include tag of view fields
sid-acryl Jun 27, 2024
36c688a
golden file test
sid-acryl Jun 28, 2024
c28e4f2
resolve merge conflict
sid-acryl Jun 28, 2024
b463ded
Update metadata-ingestion/src/datahub/ingestion/source/looker/lookml_…
sid-acryl Jul 1, 2024
094ad86
Merge branch 'master' into cus2041-import-view-field-tags
sid-acryl Jul 1, 2024
954e544
address review comments
sid-acryl Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ class ViewField:
view_name: Optional[str] = None
is_primary_key: bool = False
upstream_fields: List[str] = dataclasses_field(default_factory=list)
tags: List[str] = dataclasses_field(default_factory=list)


@dataclass
Expand Down Expand Up @@ -561,21 +562,30 @@ def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:

@staticmethod
def _get_tags_from_field_type(
field_type: ViewFieldType, reporter: SourceReport
field: ViewField, reporter: SourceReport
) -> Optional[GlobalTagsClass]:
if field_type in LookerUtil.type_to_tag_map:
return GlobalTagsClass(
tags=[
schema_field_tags: List[TagAssociationClass] = [
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
for tag_name in field.tags
]

if field.field_type in LookerUtil.type_to_tag_map:
schema_field_tags.extend(
[
TagAssociationClass(tag=tag_name)
for tag_name in LookerUtil.type_to_tag_map[field_type]
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
]
)
else:
reporter.report_warning(
"lookml",
f"Failed to map view field type {field_type}. Won't emit tags for it",
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
)
return None

if schema_field_tags:
return GlobalTagsClass(tags=schema_field_tags)

return None
Comment on lines +565 to +588
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Simplify tag extraction logic.

The tag extraction logic can be simplified by combining the user-defined and predefined tags in a single list comprehension.

    @staticmethod
    def _get_tags_from_field_type(
        field: ViewField, reporter: SourceReport
    ) -> Optional[GlobalTagsClass]:
        schema_field_tags: List[TagAssociationClass] = [
            TagAssociationClass(tag=builder.make_tag_urn(tag_name))
            for tag_name in field.tags
        ]

        if field.field_type in LookerUtil.type_to_tag_map:
            schema_field_tags.extend(
                TagAssociationClass(tag=tag_name)
                for tag_name in LookerUtil.type_to_tag_map[field.field_type]
            )
        else:
            reporter.report_warning(
                "lookml",
                f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
            )

        if schema_field_tags:
            return GlobalTagsClass(tags=schema_field_tags)

        return None
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
field: ViewField, reporter: SourceReport
) -> Optional[GlobalTagsClass]:
if field_type in LookerUtil.type_to_tag_map:
return GlobalTagsClass(
tags=[
schema_field_tags: List[TagAssociationClass] = [
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
for tag_name in field.tags
]
if field.field_type in LookerUtil.type_to_tag_map:
schema_field_tags.extend(
[
TagAssociationClass(tag=tag_name)
for tag_name in LookerUtil.type_to_tag_map[field_type]
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
]
)
else:
reporter.report_warning(
"lookml",
f"Failed to map view field type {field_type}. Won't emit tags for it",
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
)
return None
if schema_field_tags:
return GlobalTagsClass(tags=schema_field_tags)
return None
@staticmethod
def _get_tags_from_field_type(
field: ViewField, reporter: SourceReport
) -> Optional[GlobalTagsClass]:
schema_field_tags: List[TagAssociationClass] = [
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
for tag_name in field.tags
]
if field.field_type in LookerUtil.type_to_tag_map:
schema_field_tags.extend(
TagAssociationClass(tag=tag_name)
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
)
else:
reporter.report_warning(
"lookml",
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
)
if schema_field_tags:
return GlobalTagsClass(tags=schema_field_tags)
return None


@staticmethod
def get_tag_mces() -> Iterable[MetadataChangeEvent]:
Expand All @@ -602,7 +612,7 @@ def view_field_to_schema_field(
else f"{field.field_type.value}. {field.description}"
),
globalTags=(
LookerUtil._get_tags_from_field_type(field.field_type, reporter)
LookerUtil._get_tags_from_field_type(field, reporter)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Improve readability with ternary operators.

Use ternary operators to improve readability for the description and globalTags fields.

    @staticmethod
    def view_field_to_schema_field(
        field: ViewField,
        reporter: SourceReport,
        tag_measures_and_dimensions: bool = True,
    ) -> SchemaField:
        return SchemaField(
            fieldPath=field.name,
            type=LookerUtil._get_field_type(field.type, reporter),
            nativeDataType=field.type,
            label=field.label,
            description=(
                field.description
                if tag_measures_and_dimensions
                else f"{field.field_type.value}. {field.description}"
            ),
            globalTags=(
                LookerUtil._get_tags_from_field_type(field, reporter)
                if tag_measures_and_dimensions
                else None
            ),
            isPartOfKey=field.is_primary_key,
        )
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
LookerUtil._get_tags_from_field_type(field, reporter)
LookerUtil._get_tags_from_field_type(field, reporter)
@staticmethod
def view_field_to_schema_field(
field: ViewField,
reporter: SourceReport,
tag_measures_and_dimensions: bool = True,
) -> SchemaField:
return SchemaField(
fieldPath=field.name,
type=LookerUtil._get_field_type(field.type, reporter),
nativeDataType=field.type,
label=field.label,
description=(
field.description
if tag_measures_and_dimensions
else f"{field.field_type.value}. {field.description}"
),
globalTags=(
LookerUtil._get_tags_from_field_type(field, reporter)
if tag_measures_and_dimensions
else None
),
isPartOfKey=field.is_primary_key,
)

if tag_measures_and_dimensions is True
else None
),
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,7 @@ def _get_fields(
is_primary_key=is_primary_key,
field_type=type_cls,
upstream_fields=upstream_fields,
tags=field_dict.get("tags") or [],
)
fields.append(field)
return fields
Expand Down
12 changes: 0 additions & 12 deletions metadata-ingestion/tests/integration/looker/golden_looker_mces.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -140,9 +137,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -178,9 +172,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -222,9 +213,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -386,9 +383,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
22 changes: 8 additions & 14 deletions metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -386,9 +383,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -813,8 +807,8 @@
}
},
{
"entityType": "container",
anshbansal marked this conversation as resolved.
Show resolved Hide resolved
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -830,8 +824,8 @@
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -847,8 +841,8 @@
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"entityType": "container",
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -864,8 +858,8 @@
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -113,9 +110,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
21 changes: 20 additions & 1 deletion metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLanguage": "lookml"
}
},
Expand Down Expand Up @@ -338,6 +338,9 @@
"recursive": false,
"globalTags": {
"tags": [
{
"tag": "urn:li:tag:softVersion"
},
{
"tag": "urn:li:tag:Dimension"
}
Expand Down Expand Up @@ -484,5 +487,21 @@
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:softVersion",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "softVersion"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
}
]
9 changes: 0 additions & 9 deletions metadata-ingestion/tests/integration/lookml/expected_output.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1388,9 +1388,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
},
{
Expand All @@ -1405,9 +1402,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
},
{
Expand All @@ -1422,9 +1416,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
],
Expand Down
Loading
Loading