Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(gms): Return empty snapshots when one does not exist #2646

Merged
merged 8 commits into from
Jun 4, 2021
Merged

fix(gms): Return empty snapshots when one does not exist #2646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
65ecb73
Return shell entities when they don't exist
jjoyce0510 Jun 4, 2021
664d59e
Merge remote-tracking branch 'acryl/ReturnEmptySnapshots' into Return…
jjoyce0510 Jun 4, 2021
4ecedcb
fix checkstyle
jjoyce0510 Jun 4, 2021
2aa5c88
Merge remote-tracking branch 'acryl/ReturnEmptySnapshots' into Return…
jjoyce0510 Jun 4, 2021
8cd7892
Fixing tests
jjoyce0510 Jun 4, 2021
b272cc2
Merge remote-tracking branch 'acryl/ReturnEmptySnapshots' into Return…
jjoyce0510 Jun 4, 2021
1480fc3
Formatting reverts
jjoyce0510 Jun 4, 2021
3be74e8
Merge remote-tracking branch 'acryl/ReturnEmptySnapshots' into Return…
jjoyce0510 Jun 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,17 @@ public Entity getEntity(@Nonnull final Urn urn, @Nonnull final Set<String> aspec
.orElse(null);
}

public Map<Urn, Entity> getEntities(@Nonnull final Set<Urn> urns, @Nonnull final Set<String> aspectNames) {
/**
* Retrieves multiple entities.
*
* @param urns set of urns to fetch
* @param aspectNames set of aspects to fetch
* @return a map of {@link Urn} to {@link Entity} object
*/
public Map<Urn, Entity> getEntities(@Nonnull final Set<Urn> urns, @Nonnull Set<String> aspectNames) {
if (urns.isEmpty()) {
return Collections.emptyMap();
}
return getSnapshotUnions(urns, aspectNames).entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> toEntity(entry.getValue())));
}
Expand Down
26 changes: 16 additions & 10 deletions metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,29 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(@Nonnull final Set<Urn> u
final Set<EbeanAspectV2.PrimaryKey> dbKeys = urns.stream()
.map(urn -> {
final Set<String> aspectsToFetch = aspectNames.isEmpty()
? getEntityAspectNames(urn)
: aspectNames;
? getEntityAspectNames(urn)
: aspectNames;
return aspectsToFetch.stream()
.map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION))
.collect(Collectors.toList());
.map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION))
.collect(Collectors.toList());
})
.flatMap(List::stream)
.collect(Collectors.toSet());

// Fetch from db and populate urn -> aspect map.
final Map<Urn, List<RecordTemplate>> urnToAspects = new HashMap<>();

// Each urn should have some result, regardless of whether aspects are found in the DB.
for (Urn urn: urns) {
urnToAspects.putIfAbsent(urn, new ArrayList<>());
}

// Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect.
urnToAspects.keySet().forEach(key -> {
final RecordTemplate keyAspect = buildKeyAspect(key);
urnToAspects.get(key).add(keyAspect);
Comment on lines +76 to +78
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of curiosity, did this need to get shifted up?

});

_entityDao.batchGet(dbKeys).forEach((key, aspectEntry) -> {
final Urn urn = toUrn(key.getUrn());
final String aspectName = key.getAspect();
Expand All @@ -74,12 +86,6 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(@Nonnull final Set<Urn> u
urnToAspects.get(urn).add(aspectRecord);
});

// Add "key" aspects to any non null keys.
urnToAspects.keySet().forEach(key -> {
final RecordTemplate keyAspect = buildKeyAspect(key);
urnToAspects.get(key).add(keyAspect);
});

return urnToAspects;
}

Expand Down
12 changes: 6 additions & 6 deletions metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,12 @@ public void testIngestGetEntity() throws Exception {
assertEquals(2, readEntity.getValue().getCorpUserSnapshot().getAspects().size()); // Key + Info aspect.
assertTrue(DataTemplateUtil.areEqual(
writeEntity.getValue().getCorpUserSnapshot().getAspects().get(0),
readEntity.getValue().getCorpUserSnapshot().getAspects().get(0)));
readEntity.getValue().getCorpUserSnapshot().getAspects().get(1)));
CorpUserKey expectedKey = new CorpUserKey();
expectedKey.setUsername("test");
assertTrue(DataTemplateUtil.areEqual(
expectedKey,
readEntity.getValue().getCorpUserSnapshot().getAspects().get(1).getCorpUserKey()
readEntity.getValue().getCorpUserSnapshot().getAspects().get(0).getCorpUserKey()
)); // Key + Info aspect.

verify(_mockProducer, times(1)).produceMetadataAuditEvent(
Expand Down Expand Up @@ -108,25 +108,25 @@ public void testIngestGetEntities() throws Exception {
assertEquals(2, readEntity1.getValue().getCorpUserSnapshot().getAspects().size()); // Key + Info aspect.
assertTrue(DataTemplateUtil.areEqual(
writeEntity1.getValue().getCorpUserSnapshot().getAspects().get(0),
readEntity1.getValue().getCorpUserSnapshot().getAspects().get(0)));
readEntity1.getValue().getCorpUserSnapshot().getAspects().get(1)));
CorpUserKey expectedKey1 = new CorpUserKey();
expectedKey1.setUsername("tester1");
assertTrue(DataTemplateUtil.areEqual(
expectedKey1,
readEntity1.getValue().getCorpUserSnapshot().getAspects().get(1).getCorpUserKey()
readEntity1.getValue().getCorpUserSnapshot().getAspects().get(0).getCorpUserKey()
)); // Key + Info aspect.

// Entity 2
com.linkedin.entity.Entity readEntity2 = readEntities.get(entityUrn2);
assertEquals(2, readEntity2.getValue().getCorpUserSnapshot().getAspects().size()); // Key + Info aspect.
assertTrue(DataTemplateUtil.areEqual(
writeEntity2.getValue().getCorpUserSnapshot().getAspects().get(0),
readEntity2.getValue().getCorpUserSnapshot().getAspects().get(0)));
readEntity2.getValue().getCorpUserSnapshot().getAspects().get(1)));
CorpUserKey expectedKey2 = new CorpUserKey();
expectedKey2.setUsername("tester2");
assertTrue(DataTemplateUtil.areEqual(
expectedKey2,
readEntity2.getValue().getCorpUserSnapshot().getAspects().get(1).getCorpUserKey()
readEntity2.getValue().getCorpUserSnapshot().getAspects().get(0).getCorpUserKey()
)); // Key + Info aspect.

verify(_mockProducer, times(1)).produceMetadataAuditEvent(
Expand Down