Skip to content

Commit

Permalink
Adds ability for midtier to serve custom dataset properties from aspe…
Browse files Browse the repository at this point in the history 
…ct (#1881)
  • Loading branch information
Charlie Tran authored Sep 20, 2020
1 parent 205a6c9 commit 57fdc5c
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 2 deletions.
4 changes: 3 additions & 1 deletion datahub-dao/src/main/java/com/linkedin/datahub/models/view/DatasetView.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.linkedin.datahub.models.view;

import java.util.List;
import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;


@Data
@NoArgsConstructor
public class DatasetView {
Expand Down Expand Up @@ -36,4 +36,6 @@ public class DatasetView {
private Long createdTime;

private Long modifiedTime;

private Map<String, String> customProperties;
}
3 changes: 3 additions & 0 deletions datahub-dao/src/main/java/com/linkedin/datahub/util/DatasetUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public static DatasetView toDatasetView(Dataset dataset) {
if (dataset.hasLastModified()) {
view.setModifiedTime(dataset.getLastModified().getTime());
}
if (dataset.hasProperties()) {
view.setCustomProperties(dataset.getProperties());
}
return view;
}

Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/mce-cli/bootstrap_mce.dat
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot", {"urn": "urn:li:corpuser:datahub", "aspects": [("com.linkedin.pegasus2avro.identity.CorpUserInfo", {"active": True, "displayName": "Data Hub", "fullName": "Data Hub", "email": "[email protected]", "title": "CEO"})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot", {"urn": "urn:li:corpuser:jdoe", "aspects": [("com.linkedin.pegasus2avro.identity.CorpUserInfo", {"active": True, "displayName": "John Doe", "fullName": "John Doe", "email": "[email protected]", "title": "Software Engineer"})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleKafkaSchema","platform":"urn:li:dataPlatform:kafka","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.dataset.DatasetProperties", {"tags":["myfaketags"],"customProperties":{"prop1":"fakeprop","prop2":"pikachu"}}), ("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleKafkaSchema","platform":"urn:li:dataPlatform:kafka","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHdfsSchema","platform":"urn:li:dataPlatform:hdfs","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHiveSchema","platform":"urn:li:dataPlatform:hive","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DataProcessSnapshot", {"urn": "urn:li:dataProcess:(sqoop,DEMO,PROD)", "aspects": [{"owners":[{"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:datahub"}},{"inputs":["urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)","urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)"],"outputs":["urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)","urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)"]}]}), "proposedDelta": None}

0 comments on commit 57fdc5c

Please sign in to comment.