Skip to content

Commit

Permalink
AVRO-4094: [C#] Updating mapped namespaces referenced in types
Browse files Browse the repository at this point in the history
  • Loading branch information
@Shawson
Shawson committed Nov 28, 2024
1 parent 4f61326 commit b910a5b
Show file tree
Hide file tree
Showing 2 changed files with 179 additions and 0 deletions.
108 changes: 108 additions & 0 deletions lang/csharp/src/apache/main/CodeGen/CodeGen.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Avro.IO.Parsing;
using Microsoft.CSharp;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace Avro
{
Expand Down Expand Up @@ -113,6 +116,8 @@ public virtual void AddProtocol(string protocolText, IEnumerable<KeyValuePair<st
{
// Map namespaces
protocolText = ReplaceMappedNamespacesInSchema(protocolText, namespaceMapping);
protocolText = ReplaceMappedNamespacesInSchemaTypes(protocolText, namespaceMapping);

Protocol protocol = Protocol.Parse(protocolText);
Protocols.Add(protocol);
}
Expand All @@ -135,6 +140,8 @@ public virtual void AddSchema(string schemaText, IEnumerable<KeyValuePair<string
{
// Map namespaces
schemaText = ReplaceMappedNamespacesInSchema(schemaText, namespaceMapping);
schemaText = ReplaceMappedNamespacesInSchemaTypes(schemaText, namespaceMapping);

Schema schema = Schema.Parse(schemaText);
Schemas.Add(schema);
}
Expand Down Expand Up @@ -1266,5 +1273,106 @@ private static string ReplaceMappedNamespacesInSchema(string input, IEnumerable<
return $@"""namespace""{m.Groups[1].Value}:{m.Groups[2].Value}""{ns}""";
});
}
/// <summary>
/// Replace namespaces in a parsed JSON schema object for all "type" fields.
/// </summary>
/// <param name="schemaJson">The JSON schema as a string.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
/// <returns>The updated JSON schema as a string.</returns>
private static string ReplaceMappedNamespacesInSchemaTypes(string schemaJson, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
if (string.IsNullOrWhiteSpace(schemaJson) || namespaceMapping == null)
return schemaJson;

// Parse the JSON schema into a JToken object
var schemaToken = JToken.Parse(schemaJson);

// Recursively update the namespaces in the schema
UpdateNamespacesInJToken(schemaToken, namespaceMapping);

// Convert the updated schema back to a JSON string
return schemaToken.ToString(Formatting.Indented);
}

/// <summary>
/// Recursively navigates and updates "type" fields in a JToken.
/// </summary>
/// <param name="token">The current JToken to process.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
private static void UpdateNamespacesInJToken(JToken token, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
if (token is JObject obj)
{
if (obj.ContainsKey("type"))
{
var typeToken = obj["type"];
if (typeToken is JValue) // Single type
{
string type = typeToken.ToString();
obj["type"] = ReplaceNamespace(type, namespaceMapping);
}
else if (typeToken is JArray typeArray) // Array of types
{
for (int i = 0; i < typeArray.Count; i++)
{
var arrayItem = typeArray[i];
if (arrayItem is JValue) // Simple type
{
string type = arrayItem.ToString();
typeArray[i] = ReplaceNamespace(type, namespaceMapping);
}
else if (arrayItem is JObject nestedObj) // Nested object
{
UpdateNamespacesInJToken(nestedObj, namespaceMapping);
}
}
}
else if (typeToken is JObject nestedTypeObj) // Complex type
{
UpdateNamespacesInJToken(nestedTypeObj, namespaceMapping);
}
}

// Recurse into all properties of the object
foreach (var property in obj.Properties())
{
UpdateNamespacesInJToken(property.Value, namespaceMapping);
}
}
else if (token is JArray array)
{
// Recurse into all elements of the array
foreach (var element in array)
{
UpdateNamespacesInJToken(element, namespaceMapping);
}
}
}

/// <summary>
/// Replace a namespace in a string based on the provided mapping.
/// </summary>
/// <param name="originalNamespace">The original namespace string.</param>
/// <param name="namespaceMapping">The mapping of old namespaces to new namespaces.</param>
/// <returns>The updated namespace string.</returns>
private static string ReplaceNamespace(string originalNamespace, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
{
foreach (var mapping in namespaceMapping)
{
if (originalNamespace == mapping.Key)
{
return mapping.Value;
}
else if (originalNamespace.StartsWith($"{mapping.Key}."))
{
return $"{mapping.Value}.{originalNamespace.Substring(mapping.Key.Length + 1)}";
}
}
return originalNamespace;
}

}
}
71 changes: 71 additions & 0 deletions lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,46 @@ class AvroGenSchemaTests
]
}";

private const string _fullyQualifiedTypeReferences = @"
[
{
""namespace"": ""org.apache.avro.codegentest.testdata.common"",
""type"": ""enum"",
""name"": ""Planet"",
""doc"" : ""Test mapping of types in other namespaces post-map"",
""symbols"": [
""Mercury"",
""Venus"",
""Earth"",
""Mars"",
""Jupiter"",
""Saturn"",
""Neptune"",
""Uranus""
],
},
{
""namespace"": ""org.apache.avro.codegentest.testdata.users"",
""type"": ""record"",
""name"": ""User"",
""doc"" : ""Test mapping of types in other namespaces post-map"",
""fields"": [
{
""name"": ""homePlanet"",
""type"": ""org.apache.avro.codegentest.testdata.common.Planet""
},
{
""name"": ""favouritePlanet"",
""type"": [
""null"",
""org.apache.avro.codegentest.testdata.common.Planet""
],
""default"": null
}]
},
]";

private Assembly TestSchema(
string schema,
IEnumerable<string> typeNamesToCheck = null,
Expand Down Expand Up @@ -606,6 +646,37 @@ public void GenerateSchemaWithNamespaceMapping(
AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary<string, string> { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck);
}

[TestCase(_fullyQualifiedTypeReferences,
new string[]
{
"org.apache.avro.codegentest.testdata.common:Test.Common",
"org.apache.avro.codegentest.testdata.users:Test.Users"
},
new string[]
{
"Test.Common.Planet",
"Test.Users.User",
},
new string[]
{
"Test/Common/Planet.cs",
"Test/Users/User.cs"
})]
public void GenerateSchemaWithMultipleNamespaceMapping(
string schema,
IEnumerable<string> namespaceMappings,
IEnumerable<string> typeNamesToCheck,
IEnumerable<string> generatedFilesToCheck)
{
var namespaceMappingsDict = new Dictionary<string, string>();

foreach(var mapping in namespaceMappings)
{
namespaceMappingsDict.Add(mapping.Split(':')[0], mapping.Split(':')[1]);
}
AvroGenHelper.TestSchema(schema, typeNamesToCheck, namespaceMappingsDict, generatedFilesToCheck);
}

[TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))]
[TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))]
public void NotSupportedSchema(string schema, Type expectedException)
Expand Down

0 comments on commit b910a5b

Please sign in to comment.