Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ingest/lookml): ingest field tags #10792

Merged
merged 6 commits into from
Jul 1, 2024
Merged

feat(ingest/lookml): ingest field tags #10792

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
27dbba4
include tag of view fields
sid-acryl Jun 27, 2024
36c688a
golden file test
sid-acryl Jun 28, 2024
c28e4f2
resolve merge conflict
sid-acryl Jun 28, 2024
b463ded
Update metadata-ingestion/src/datahub/ingestion/source/looker/lookml_…
sid-acryl Jul 1, 2024
094ad86
Merge branch 'master' into cus2041-import-view-field-tags
sid-acryl Jul 1, 2024
954e544
address review comments
sid-acryl Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ class ViewField:
view_name: Optional[str] = None
is_primary_key: bool = False
upstream_fields: List[str] = dataclasses_field(default_factory=list)
tags: List[str] = dataclasses_field(default_factory=list)


@dataclass
Expand Down Expand Up @@ -561,21 +562,35 @@ def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:

@staticmethod
def _get_tags_from_field_type(
field_type: ViewFieldType, reporter: SourceReport
field: ViewField, reporter: SourceReport
) -> Optional[GlobalTagsClass]:
if field_type in LookerUtil.type_to_tag_map:
return GlobalTagsClass(
tags=[
schema_field_tags: List[TagAssociationClass] = []

# Add any user defined tag present on field
schema_field_tags.extend(
sid-acryl marked this conversation as resolved.
Show resolved Hide resolved
[
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
for tag_name in field.tags
]
)

if field.field_type in LookerUtil.type_to_tag_map:
schema_field_tags.extend(
[
TagAssociationClass(tag=tag_name)
for tag_name in LookerUtil.type_to_tag_map[field_type]
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
]
)
else:
reporter.report_warning(
"lookml",
f"Failed to map view field type {field_type}. Won't emit tags for it",
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
)
return None

if schema_field_tags:
return GlobalTagsClass(tags=schema_field_tags)

return None
sid-acryl marked this conversation as resolved.
Show resolved Hide resolved

@staticmethod
def get_tag_mces() -> Iterable[MetadataChangeEvent]:
Expand All @@ -602,7 +617,7 @@ def view_field_to_schema_field(
else f"{field.field_type.value}. {field.description}"
),
globalTags=(
LookerUtil._get_tags_from_field_type(field.field_type, reporter)
LookerUtil._get_tags_from_field_type(field, reporter)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Improve readability with ternary operators.

Use ternary operators to improve readability for the description and globalTags fields.

    @staticmethod
    def view_field_to_schema_field(
        field: ViewField,
        reporter: SourceReport,
        tag_measures_and_dimensions: bool = True,
    ) -> SchemaField:
        return SchemaField(
            fieldPath=field.name,
            type=LookerUtil._get_field_type(field.type, reporter),
            nativeDataType=field.type,
            label=field.label,
            description=(
                field.description
                if tag_measures_and_dimensions
                else f"{field.field_type.value}. {field.description}"
            ),
            globalTags=(
                LookerUtil._get_tags_from_field_type(field, reporter)
                if tag_measures_and_dimensions
                else None
            ),
            isPartOfKey=field.is_primary_key,
        )
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
LookerUtil._get_tags_from_field_type(field, reporter)
LookerUtil._get_tags_from_field_type(field, reporter)
@staticmethod
def view_field_to_schema_field(
field: ViewField,
reporter: SourceReport,
tag_measures_and_dimensions: bool = True,
) -> SchemaField:
return SchemaField(
fieldPath=field.name,
type=LookerUtil._get_field_type(field.type, reporter),
nativeDataType=field.type,
label=field.label,
description=(
field.description
if tag_measures_and_dimensions
else f"{field.field_type.value}. {field.description}"
),
globalTags=(
LookerUtil._get_tags_from_field_type(field, reporter)
if tag_measures_and_dimensions
else None
),
isPartOfKey=field.is_primary_key,
)

if tag_measures_and_dimensions is True
else None
),
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,7 @@ def _get_fields(
is_primary_key=is_primary_key,
field_type=type_cls,
upstream_fields=upstream_fields,
tags=field_dict["tags"] if field_dict.get("tags") is not None else [],
sid-acryl marked this conversation as resolved.
Show resolved Hide resolved
)
fields.append(field)
return fields
Expand Down
12 changes: 0 additions & 12 deletions metadata-ingestion/tests/integration/looker/golden_looker_mces.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -174,9 +168,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -218,9 +209,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
22 changes: 8 additions & 14 deletions metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -783,8 +777,8 @@
}
},
{
"entityType": "container",
anshbansal marked this conversation as resolved.
Show resolved Hide resolved
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -799,8 +793,8 @@
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -815,8 +809,8 @@
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"entityType": "container",
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -831,8 +825,8 @@
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -113,9 +110,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
21 changes: 20 additions & 1 deletion metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLanguage": "lookml"
}
},
Expand Down Expand Up @@ -338,6 +338,9 @@
"recursive": false,
"globalTags": {
"tags": [
{
"tag": "urn:li:tag:softVersion"
},
{
"tag": "urn:li:tag:Dimension"
}
Expand Down Expand Up @@ -484,5 +487,21 @@
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:softVersion",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "softVersion"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
}
]
Loading