Skip to content

Commit

Permalink
[TEMP] feat(ConfigForm): Add support for oneOf fields
Browse files Browse the repository at this point in the history 
Currently, we have partial support for the `oneOf` schemas.

This commit cleans up the generated schemas, by removing the `not`
definition from `oneOf` and also removing the empty properties like:

'''
{
  property: { }
}
'''

fix: KaotoIO#1550
fix: KaotoIO#948
  • Loading branch information
@lordrip
lordrip committed Oct 9, 2024
1 parent bce3f44 commit 96eddb2
Show file tree
Hide file tree
Showing 14 changed files with 702 additions and 281 deletions.
134 changes: 98 additions & 36 deletions ...-generator/src/main/java/io/kaoto/camelcatalog/generator/CamelYamlDslSchemaProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,42 +15,34 @@
*/
package io.kaoto.camelcatalog.generator;

import java.io.StringWriter;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

import java.io.StringWriter;
import java.util.*;

/**
* Process camelYamlDsl.json file, aka Camel YAML DSL JSON schema.
*/
public class CamelYamlDslSchemaProcessor {
private static final String PROCESSOR_DEFINITION = "org.apache.camel.model.ProcessorDefinition";
private static final String TOKENIZER_DEFINITION = "org.apache.camel.model.TokenizerDefinition";
private static final String ROUTE_CONFIGURATION_DEFINITION = "org.apache.camel.model.RouteConfigurationDefinition";
private static final String LOAD_BALANCE_DEFINITION = "org.apache.camel.model.LoadBalanceDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION = "org.apache.camel.model.ExpressionSubElementDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION =
"org.apache.camel.model.ExpressionSubElementDefinition";
private static final String SAGA_DEFINITION = "org.apache.camel.model.SagaDefinition";
private static final String PROPERTY_EXPRESSION_DEFINITION = "org.apache.camel.model.PropertyExpressionDefinition";
private static final String ERROR_HANDLER_DEFINITION = "org.apache.camel.model.ErrorHandlerDefinition";
private static final String ERROR_HANDLER_DESERIALIZER = "org.apache.camel.dsl.yaml.deserializers.ErrorHandlerBuilderDeserializer";
private static final String ERROR_HANDLER_DESERIALIZER =
"org.apache.camel.dsl.yaml.deserializers.ErrorHandlerBuilderDeserializer";
private final ObjectMapper jsonMapper;
private final ObjectNode yamlDslSchema;
private final List<String> processorBlocklist = List.of(
"org.apache.camel.model.KameletDefinition"
// reactivate entries once we have a better handling of how to add WHEN and
// OTHERWISE without Catalog
// "Otherwise",
// "when",
// "doCatch",
// ""doFinally"
);
private final List<String> processorBlocklist = List.of("org.apache.camel.model.KameletDefinition");

/**
* The processor properties those should be handled separately, i.e. remove from
Expand All @@ -67,8 +59,7 @@ public class CamelYamlDslSchemaProcessor {
List.of("uri", "parameters"),
"org.apache.camel.model.WireTapDefinition",
List.of("uri", "parameters"));
private final List<String> processorReferenceBlockList = List.of(
PROCESSOR_DEFINITION);
private final List<String> processorReferenceBlockList = List.of(PROCESSOR_DEFINITION);

public CamelYamlDslSchemaProcessor(ObjectMapper mapper, ObjectNode yamlDslSchema) throws Exception {
this.jsonMapper = mapper;
Expand Down Expand Up @@ -107,11 +98,14 @@ private String doProcessSubSchema(
var answer = (ObjectNode) prop.getValue().deepCopy();
if (answer.has("$ref") && definitions.has(getNameFromRef(answer))) {
answer = definitions.withObject("/" + getNameFromRef(answer)).deepCopy();

}

extractSingleOneOfFromAnyOf(answer);
removeEmptyProperties(answer);
removeNotFromOneOf(answer);
answer.set("$schema", rootSchema.get("$schema"));
populateDefinitions(answer, definitions);

var writer = new StringWriter();
try {
JsonGenerator gen = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
Expand All @@ -122,6 +116,58 @@ private String doProcessSubSchema(
}
}

/**
* Remove the empty properties from the definition, as it's not supported by the
* form library. This happens when the properties are supposed to be handled
* separately, in a oneOf definition, for example.
* An example:
* ```
* {
* properties: {
* deadLetterChannel: { },
* defaultErrorHandler: { }
* }
* }
*
* @param definitionWithEmptyProperties the definition containing the empty properties to be removed
*/
private void removeEmptyProperties(ObjectNode definitionWithEmptyProperties) {
if (!definitionWithEmptyProperties.has("properties")) {
return;
}

var propertiesObject = definitionWithEmptyProperties.withObject("/properties");
List<String> propToRemove = new ArrayList<>();
propertiesObject.fields().forEachRemaining(field -> {
if (field.getValue().isEmpty()) {
propToRemove.add(field.getKey());
}
});
propToRemove.forEach(propertiesObject::remove);
}

/**
* Remove "not" from the OneOf definition, as it's not supported by the
* form library. The "not" is only useful for the Source code editor.
*
* @param definitionContainingOneOf the definition to be cleaned
*/
private void removeNotFromOneOf(ObjectNode definitionContainingOneOf) {
if (!definitionContainingOneOf.has("oneOf")) {
return;
}

ArrayNode cleanAnyOf = jsonMapper.createArrayNode();
var oneOfDefinitionArray = definitionContainingOneOf.withArray("/oneOf");
for (var def : oneOfDefinitionArray) {
if (def.has("not")) {
continue;
}
cleanAnyOf.add(def);
}
definitionContainingOneOf.set("oneOf", cleanAnyOf);
}

private String getNameFromRef(ObjectNode parent) {
var ref = parent.get("$ref").asText();
return ref.contains("items") ? ref.replace("#/items/definitions/", "")
Expand Down Expand Up @@ -152,9 +198,7 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
* root definitions.
* It's a workaround for the current Camel YAML DSL JSON schema, where some
* AnyOf definition
* contains only one OneOf definition. This can be removed once
* https://github.com/KaotoIO/kaoto/issues/948
* is resolved.
* contains only one OneOf definition.
* This is done mostly for the errorHandler definition, f.i.
* ```
* {
Expand Down Expand Up @@ -214,7 +258,7 @@ private void extractSingleOneOfFromAnyOf(ObjectNode definition) {
* "$comment" in the property schema</li>
* </ul>
*
* @return
* @return A map of processor definitions
*/
public Map<String, ObjectNode> getProcessors() throws Exception {
var definitions = yamlDslSchema
Expand All @@ -235,8 +279,16 @@ public Map<String, ObjectNode> getProcessors() throws Exception {
var processor = relocatedDefinitions.withObject("/" + processorFQCN);
processor = extractFromOneOf(processorFQCN, processor);
processor.remove("oneOf");
processor = extractFromAnyOfOneOf(processorFQCN, processor);
processor.remove("anyOf");

/* Preparation for TokenizerDefinition, this could be propagated to all EIPs in the future */
if (processorFQCN.equals(TOKENIZER_DEFINITION)) {
removeEmptyProperties(processor);
extractSingleOneOfFromAnyOf(processor);
removeNotFromOneOf(processor);
}

processAndRemoveAnyOfForSubCatalogs(processorFQCN, processor);

var processorProperties = processor.withObject("/properties");
Set<String> propToRemove = new HashSet<>();
var propertyBlockList = processorPropertyBlockList.get(processorFQCN);
Expand Down Expand Up @@ -316,10 +368,12 @@ private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws E
for (var def : oneOf) {
if (def.get("type").asText().equals("object")) {
var objectDef = (ObjectNode) def;
if (definition.has("title"))
if (definition.has("title")) {
objectDef.set("title", definition.get("title"));
if (definition.has("description"))
}
if (definition.has("description")) {
objectDef.set("description", definition.get("description"));
}
return objectDef;
}
}
Expand All @@ -328,9 +382,19 @@ private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws E
name));
}

private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) throws Exception {
/**
* Process the "anyOf" definition for the sub-catalogs, such as expressions, languages,
* data formats, and errorHandler.
* It puts a "$comment" in the schema to indicate the type of the sub-catalog and then removes
* the "anyOf" definition.
*
* @param name the FQCN of the definition, for instance "org.apache.camel.model.LoadBalanceDefinition"
* @param definition the definition that potentially could have "anyOf" definition, referencing to another
* sub-catalogs
*/
private void processAndRemoveAnyOfForSubCatalogs(String name, ObjectNode definition) {
if (!definition.has("anyOf")) {
return definition;
return;
}
var anyOfOneOf = definition.withArray("/anyOf").get(0).withArray("/oneOf");
for (var def : anyOfOneOf) {
Expand All @@ -355,7 +419,6 @@ private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) thr
}
}
definition.remove("anyOf");
return definition;
}

private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) throws Exception {
Expand All @@ -373,7 +436,7 @@ private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) thr

var definition = (ObjectNode) entry.getValue();
definition = extractFromOneOf(definitionName, definition);
definition = extractFromAnyOfOneOf(definitionName, definition);
processAndRemoveAnyOfForSubCatalogs(definitionName, definition);
var definitionProperties = definition.withObject("/properties");
var propToRemove = new HashSet<String>();
for (var property : definitionProperties.properties()) {
Expand Down Expand Up @@ -529,7 +592,7 @@ public Map<String, ObjectNode> getLanguages() throws Exception {
* <li>If the processor property is expression aware, it puts "expression" as a
* "$comment" in the property schema</li>
*
* @return
* @return A map of the entity name and the schema
*/
public Map<String, ObjectNode> getEntities() throws Exception {
var definitions = yamlDslSchema
Expand All @@ -548,8 +611,7 @@ public Map<String, ObjectNode> getEntities() throws Exception {
var yamlInDefinition = relocatedDefinitions.withObject("/" + yamlInFQCN);
yamlInDefinition = extractFromOneOf(yamlInFQCN, yamlInDefinition);
yamlInDefinition.remove("oneOf");
yamlInDefinition = extractFromAnyOfOneOf(yamlInFQCN, yamlInDefinition);
yamlInDefinition.remove("anyOf");
processAndRemoveAnyOfForSubCatalogs(yamlInFQCN, yamlInDefinition);
Set<String> propToRemove = new HashSet<>();
var yamlInProperties = yamlInDefinition.withObject("/properties");
for (var yamlInPropertyEntry : yamlInProperties.properties()) {
Expand Down
57 changes: 48 additions & 9 deletions ...erator/src/test/java/io/kaoto/camelcatalog/generator/CamelYamlDslSchemaProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,23 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.camel.dsl.yaml.YamlRoutesBuilderLoader;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.util.List;

import static org.junit.jupiter.api.Assertions.*;

public class CamelYamlDslSchemaProcessorTest {
private final ObjectMapper jsonMapper;
private final ObjectNode yamlDslSchema;
private final CamelYamlDslSchemaProcessor processor;
private ObjectMapper jsonMapper;
private CamelYamlDslSchemaProcessor processor;

public CamelYamlDslSchemaProcessorTest() throws Exception {
@BeforeEach
public void setUp() throws Exception {
jsonMapper = new ObjectMapper();
var is = YamlRoutesBuilderLoader.class.getClassLoader().getResourceAsStream("schema/camelYamlDsl.json");
yamlDslSchema = (ObjectNode) jsonMapper.readTree(is);
ObjectNode yamlDslSchema = (ObjectNode) jsonMapper.readTree(is);

processor = new CamelYamlDslSchemaProcessor(jsonMapper, yamlDslSchema);
}

Expand Down Expand Up @@ -60,7 +62,41 @@ public void testExtractSingleOneOfFromAnyOf() throws Exception {

assertTrue(errorHandlerSchema.has("oneOf"));
assertTrue(errorHandlerSchema.get("oneOf").isArray());
assertEquals(7, errorHandlerSchema.get("oneOf").size());
assertEquals(6, errorHandlerSchema.get("oneOf").size());
}

@Test
public void testRemoveEmptyProperties() throws Exception {
var tokenizerRootSchema = (ObjectNode) jsonMapper.readTree(
getClass().getClassLoader().getResourceAsStream("camel-4.9.0-tokenizer-schema.json"));
processor = new CamelYamlDslSchemaProcessor(jsonMapper, tokenizerRootSchema);

var subSchemaMap = processor.processSubSchema();
var tokenizerSchema = jsonMapper.readTree(subSchemaMap.get("tokenizer"));
var properties = tokenizerSchema.withObject("/properties");

assertFalse(properties.has("langChain4jCharacterTokenizer"));
assertFalse(properties.has("langChain4jLineTokenizer"));
assertFalse(properties.has("langChain4jParagraphTokenizer"));
assertFalse(properties.has("langChain4jSentenceTokenizer"));
assertFalse(properties.has("langChain4jWordTokenizer"));
}

@Test
public void testRemoveNotFromOneOf() throws Exception {
var tokenizerRootSchema = (ObjectNode) jsonMapper.readTree(
getClass().getClassLoader().getResourceAsStream("camel-4.9.0-tokenizer-schema.json"));
processor = new CamelYamlDslSchemaProcessor(jsonMapper, tokenizerRootSchema);

var subSchemaMap = processor.processSubSchema();
var tokenizerSchema = jsonMapper.readTree(subSchemaMap.get("tokenizer"));
var oneOfArray = tokenizerSchema.withArray("/oneOf");

assertEquals(5, oneOfArray.size());
oneOfArray.forEach(oneOf -> {
assertTrue(oneOf.has("properties"));
assertFalse(oneOf.has("not"));
});
}

@Test
Expand All @@ -81,7 +117,8 @@ public void testGetDataFormats() throws Exception {
public void testGetDataFormatYaml() throws Exception {
var dataFormatMap = processor.getDataFormats();
var yamlDataFormat = dataFormatMap.get("yaml");
var typeFilterDefinition = yamlDataFormat.withObject("/definitions").withObject("org.apache.camel.model.dataformat.YAMLTypeFilterDefinition");
var typeFilterDefinition = yamlDataFormat.withObject("/definitions")
.withObject("org.apache.camel.model.dataformat.YAMLTypeFilterDefinition");
assertEquals("object", typeFilterDefinition.get("type").asText());
var propType = typeFilterDefinition.withObject("/properties").withObject("/type");
assertEquals("string", propType.get("type").asText());
Expand Down Expand Up @@ -141,9 +178,11 @@ public void testGetProcessors() throws Exception {
var actionDef = saga.withObject("/definitions").withObject("/org.apache.camel.model.SagaActionUriDefinition");
assertFalse(actionDef.has("oneOf"));
assertEquals("object", actionDef.withObject("/properties").withObject("/parameters").get("type").asText());
var propExpDef = saga.withObject("/definitions").withObject("/org.apache.camel.model.PropertyExpressionDefinition");
var propExpDef =
saga.withObject("/definitions").withObject("/org.apache.camel.model.PropertyExpressionDefinition");
assertEquals("object", propExpDef.withObject("/properties").withObject("/expression").get("type").asText());
assertEquals("expression", propExpDef.withObject("/properties").withObject("/expression").get("$comment").asText());
assertEquals("expression",
propExpDef.withObject("/properties").withObject("/expression").get("$comment").asText());
}

@Test
Expand Down
Loading

0 comments on commit 96eddb2

Please sign in to comment.