Skip to content

Commit

Permalink
Reorder schema string entry in Delta Lake
Browse files Browse the repository at this point in the history
This improves the readability when checking
transaction log's diff side-by-side between Trino and Delta Lake.

Additionally, add link to protocol.
  • Loading branch information
ebyhr committed May 10, 2023
1 parent f722f1e commit 0df1476
Showing 1 changed file with 12 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,10 @@ private static Map<String, Object> serializeStructType(
Map<String, Boolean> columnNullability,
Map<String, Map<String, Object>> columnMetadata)
{
// https://github.com/delta-io/delta/blob/master/PROTOCOL.md#struct-type
ImmutableMap.Builder<String, Object> schema = ImmutableMap.builder();

schema.put("type", "struct");
schema.put("fields", columns.stream()
.map(column -> {
String columnName = column.getColumnName();
Expand All @@ -174,15 +176,19 @@ private static Map<String, Object> serializeStructType(
columnMetadata.get(columnName));
})
.collect(toImmutableList()));
schema.put("type", "struct");

return schema.buildOrThrow();
}

private static Map<String, Object> serializeStructField(String name, Type type, @Nullable String comment, @Nullable Boolean nullable, @Nullable Map<String, Object> metadata)
{
// https://github.com/delta-io/delta/blob/master/PROTOCOL.md#struct-field
ImmutableMap.Builder<String, Object> fieldContents = ImmutableMap.builder();

fieldContents.put("name", name);
fieldContents.put("type", serializeColumnType(type));
fieldContents.put("nullable", nullable != null ? nullable : true);

ImmutableMap.Builder<String, Object> columnMetadata = ImmutableMap.builder();
if (comment != null) {
columnMetadata.put("comment", comment);
Expand All @@ -192,11 +198,7 @@ private static Map<String, Object> serializeStructField(String name, Type type,
.filter(entry -> !entry.getKey().equals("comment"))
.forEach(entry -> columnMetadata.put(entry.getKey(), entry.getValue()));
}

fieldContents.put("metadata", columnMetadata.buildOrThrow());
fieldContents.put("name", name);
fieldContents.put("nullable", nullable != null ? nullable : true);
fieldContents.put("type", serializeColumnType(type));

return fieldContents.buildOrThrow();
}
Expand All @@ -217,23 +219,25 @@ private static Object serializeColumnType(Type columnType)

private static Map<String, Object> serializeArrayType(ArrayType arrayType)
{
// https://github.com/delta-io/delta/blob/master/PROTOCOL.md#array-type
ImmutableMap.Builder<String, Object> fields = ImmutableMap.builder();

fields.put("type", "array");
fields.put("containsNull", true);
fields.put("elementType", serializeColumnType(arrayType.getElementType()));
fields.put("containsNull", true);

return fields.buildOrThrow();
}

private static Map<String, Object> serializeMapType(MapType mapType)
{
// https://github.com/delta-io/delta/blob/master/PROTOCOL.md#map-type
ImmutableMap.Builder<String, Object> fields = ImmutableMap.builder();

fields.put("keyType", serializeColumnType(mapType.getKeyType()));
fields.put("type", "map");
fields.put("valueContainsNull", true);
fields.put("keyType", serializeColumnType(mapType.getKeyType()));
fields.put("valueType", serializeColumnType(mapType.getValueType()));
fields.put("valueContainsNull", true);

return fields.buildOrThrow();
}
Expand Down

0 comments on commit 0df1476

Please sign in to comment.