Skip to content

Commit

Permalink
fix: Refactor processors schema (#531)
Browse files Browse the repository at this point in the history 
* Extract processor JSON schema from main camelYamlDsl.json and put into pattern catalog as a `/propertiesSchema`
* Add control comment for complex parameters to `$comment` schema property, such as `class:<class name>`, `expression`, `dataformat`, `loadbalance` for those need special handling for configuration form rendering
* Remove oneOf/anyOf structures and extract the one needed for form rendering
* Added exhaustive rendering test for all processors

Fixes: #448
Fixes: #449
  • Loading branch information
@igarashitm
igarashitm authored Dec 7, 2023
1 parent fadf4f5 commit e9d71c6
Show file tree
Hide file tree
Showing 13 changed files with 837 additions and 415 deletions.
86 changes: 50 additions & 36 deletions ...camel-catalog-maven-plugin/src/main/java/io/kaoto/camelcatalog/CamelCatalogProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,8 @@
import org.apache.camel.tooling.model.ComponentModel;
import org.apache.camel.tooling.model.EipModel;
import org.apache.camel.tooling.model.JsonMapper;
import org.apache.camel.util.json.JsonObject;

import java.io.StringWriter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
Expand All @@ -38,14 +36,11 @@ public class CamelCatalogProcessor {
private final ObjectMapper jsonMapper;
private final DefaultCamelCatalog api;
private final CamelYamlDslSchemaProcessor schemaProcessor;
private final ArrayList<String> patternBlocklist;

public CamelCatalogProcessor(ObjectMapper jsonMapper, CamelYamlDslSchemaProcessor schemaProcessor) {
this.jsonMapper = jsonMapper;
this.api = new DefaultCamelCatalog();
this.schemaProcessor = schemaProcessor;
patternBlocklist = new ArrayList<>();
populatePatternBlocklist();
}

/**
Expand Down Expand Up @@ -194,16 +189,26 @@ public String getLanguageCatalog() throws Exception {
}

public String getModelCatalog() throws Exception {
var answer = new LinkedHashMap<String, JsonObject>();
var answer = jsonMapper.createObjectNode();
api.findModelNames().stream().sorted().forEach((name) -> {
try {
var model = (EipModel) api.model(Kind.eip, name);
answer.put(name, JsonMapper.asJsonObject(model));
var json = JsonMapper.asJsonObject(model).toJson();
var catalogNode = (ObjectNode) jsonMapper.readTree(json);
if ("from".equals(name)) {
// "from" is an exception that is not a processor, therefore it's not in the
// pattern catalog - put the propertiesSchema here
generatePropertiesSchema(catalogNode);
}
answer.set(name, catalogNode);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
return JsonMapper.serialize(answer);
StringWriter writer = new StringWriter();
var jsonGenerator = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
jsonMapper.writeTree(jsonGenerator, answer);
return writer.toString();
}

/**
Expand All @@ -212,35 +217,44 @@ public String getModelCatalog() throws Exception {
* @throws Exception
*/
public String getPatternCatalog() throws Exception {
var answer = new LinkedHashMap<String, JsonObject>();
var answer = jsonMapper.createObjectNode();
var processors = schemaProcessor.getProcessors();
api.findModelNames().stream().sorted().forEach((name) -> {
var model = (EipModel) api.model(Kind.eip, name);
if (!processors.has(name) || patternBlocklist.contains(name)) {
return;
}
var javaType = schemaProcessor.getProcessorDefinitionFQCN(name);
if (javaType.equals(model.getJavaType())) {
answer.put(name, JsonMapper.asJsonObject(model));
var catalogMap = new LinkedHashMap<String, EipModel>();
for (var name : api.findModelNames()) {
var modelCatalog = (EipModel) api.model(Kind.eip, name);
catalogMap.put(modelCatalog.getJavaType(), modelCatalog);
}
for (var entry : processors.entrySet()) {
var processorFQCN = entry.getKey();
var processorSchema = entry.getValue();
var processorCatalog = catalogMap.get(processorFQCN);
for (var property : processorSchema.withObject("/properties").properties()) {
var propertyName = property.getKey();
var propertySchema = (ObjectNode) property.getValue();
if ("org.apache.camel.model.ToDynamicDefinition".equals(processorFQCN) && "parameters".equals(propertyName)) {
// "parameters" as a common property is omitted in the catalog, but we need this for "toD"
propertySchema.put("title", "Parameters");
propertySchema.put("description", "URI parameters");
continue;
}
var catalogOpOptional = processorCatalog.getOptions().stream().filter(op -> op.getName().equals(propertyName)).findFirst();
if (catalogOpOptional.isEmpty()) {
throw new Exception(String.format("Option '%s' not found for processor '%s'", propertyName, processorFQCN));
}
var catalogOp = catalogOpOptional.get();
if ("object".equals(catalogOp.getType()) && !catalogOp.getJavaType().startsWith("java.util.Map")
&& !propertySchema.has("$comment")) {
propertySchema.put("$comment", "class:" + catalogOp.getJavaType());
}
}
});
return JsonMapper.serialize(answer);
}

private void populatePatternBlocklist() {
this.patternBlocklist.add("kamelet");
this.patternBlocklist.add("loadBalance");
this.patternBlocklist.add("onFallback");
this.patternBlocklist.add("pipeline");
this.patternBlocklist.add("policy");
this.patternBlocklist.add("rollback");
this.patternBlocklist.add("serviceCall");
this.patternBlocklist.add("setExchangePattern");
this.patternBlocklist.add("whenSkipSendToEndpoint");
// reactivate entries once we have a better handling of how to add WHEN and OTHERWISE without Catalog
// this.patternBlocklist.add("Otherwise");
// this.patternBlocklist.add("when");
// this.patternBlocklist.add("doCatch");
// this.patternBlocklist.add("doFinally");
var json = JsonMapper.asJsonObject(processorCatalog).toJson();
var catalogTree = (ObjectNode) jsonMapper.readTree(json);
catalogTree.set("propertiesSchema", processorSchema);
answer.set(processorCatalog.getName(), catalogTree);
}
StringWriter writer = new StringWriter();
var jsonGenerator = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
jsonMapper.writeTree(jsonGenerator, answer);
return writer.toString();
}
}
200 changes: 193 additions & 7 deletions ...catalog-maven-plugin/src/main/java/io/kaoto/camelcatalog/CamelYamlDslSchemaProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,46 @@
import com.fasterxml.jackson.databind.node.ObjectNode;

import java.io.StringWriter;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Process camelYamlDsl.json file, aka Camel YAML DSL JSON schema.
*/
public class CamelYamlDslSchemaProcessor {
private static final String PROCESSOR_DEFINITION = "org.apache.camel.model.ProcessorDefinition";
private static final String LOAD_BALANCE_DEFINITION = "org.apache.camel.model.LoadBalanceDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION = "org.apache.camel.model.ExpressionSubElementDefinition";
private static final String SAGA_DEFINITION = "org.apache.camel.model.SagaDefinition";
private static final String PROPERTY_EXPRESSION_DEFINITION = "org.apache.camel.model.PropertyExpressionDefinition";
private final ObjectMapper jsonMapper;
private final ObjectNode yamlDslSchema;
private final List<String> processorBlocklist = List.of(
"org.apache.camel.model.KameletDefinition"
// reactivate entries once we have a better handling of how to add WHEN and OTHERWISE without Catalog
// "Otherwise",
// "when",
// "doCatch",
// ""doFinally"
);
/** The processor properties those should be handled separately, i.e. remove from the properties schema,
* such as branching node and parameters reflected from the underlying components. */
private final Map<String, List<String>> processorPropertyBlockList = Map.of(
"org.apache.camel.model.ChoiceDefinition",
List.of("when", "otherwise"),
"org.apache.camel.model.TryDefinition",
List.of("doCatch", "doFinally"),
"org.apache.camel.model.ToDefinition",
List.of("uri", "parameters"),
"org.apache.camel.model.WireTapDefinition",
List.of("uri", "parameters")
);
private final List<String> processorReferenceBlockList = List.of(
PROCESSOR_DEFINITION
);

public CamelYamlDslSchemaProcessor(ObjectMapper mapper, ObjectNode yamlDslSchema) throws Exception {
this.jsonMapper = mapper;
Expand Down Expand Up @@ -96,6 +127,9 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
added = false;
for (JsonNode refParent : schema.findParents("$ref")) {
var name = getNameFromRef((ObjectNode) refParent);
if (processorReferenceBlockList.contains(name)) {
continue;
}
if (!schema.has("definitions") || !schema.withObject("/definitions").has(name)) {
var schemaDefinitions = schema.withObject("/definitions");
schemaDefinitions.set(name, definitions.withObject("/" + name));
Expand All @@ -106,19 +140,171 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
}
}

public ObjectNode getProcessors() {
return yamlDslSchema
/**
* Extract the processor definitions from the main Camel YAML DSL JSON schema in the usable
* format for uniforms to render the configuration form. It does a couple of things:
* <ul>
* <li>Remove "oneOf" and "anyOf"</li>
* <li>Remove properties those are supposed to be handled separately:
* <ul>
* <li>"steps": branching steps</li>
* <li>"parameters": component parameters</li>
* <li>expression languages</li>
* <li>dataformats</li>
* </ul></li>
* <li>If the processor is expression aware, it puts "expression" as a "$comment" in the schema</li>
* <li>If the processor is dataformat aware, it puts "dataformat" as a "$comment" in the schema</li>
* <li>If the processor property is expression aware, it puts "expression" as a "$comment" in the property schema</li>
* </ul>
* @return
*/
public Map<String, ObjectNode> getProcessors() throws Exception {
var definitions = yamlDslSchema
.withObject("/items")
.withObject("/definitions")
.withObject("/org.apache.camel.model.ProcessorDefinition")
.withObject("/definitions");
var relocatedDefinitions = relocateToRootDefinitions(definitions);
var processors = relocatedDefinitions
.withObject(PROCESSOR_DEFINITION)
.withObject("/properties");

var answer = new LinkedHashMap<String, ObjectNode>();
for( var processorEntry : processors) {
var processorFQCN = getNameFromRef((ObjectNode)processorEntry);
if (processorBlocklist.contains(processorFQCN)) {
continue;
}
var processor = relocatedDefinitions.withObject("/" + processorFQCN);
processor = extractFromOneOf(processorFQCN, processor);
processor.remove("oneOf");
processor = extractFromAnyOfOneOf(processorFQCN, processor);
processor.remove("anyOf");
var processorProperties = processor.withObject("/properties");
Set<String> propToRemove = new HashSet<>();
var propertyBlockList = processorPropertyBlockList.get(processorFQCN);
for (var propEntry : processorProperties.properties()) {
var propName = propEntry.getKey();
if (propertyBlockList != null && propertyBlockList.contains(propName)) {
propToRemove.add(propName);
continue;
}
if (!LOAD_BALANCE_DEFINITION.equals(processorFQCN) && propName.equals("inheritErrorHandler")) {
// workaround for https://issues.apache.org/jira/browse/CAMEL-20188
// TODO remove this once updated to camel 4.3.0
propToRemove.add(propName);
continue;
}
var property = (ObjectNode) propEntry.getValue();
var refParent = property.findParent("$ref");
if (refParent != null) {
var ref = getNameFromRef(refParent);
if (processorReferenceBlockList.contains(ref)) {
propToRemove.add(propName);
}
if (EXPRESSION_SUB_ELEMENT_DEFINITION.equals(ref)) {
refParent.remove("$ref");
refParent.put("type", "object");
refParent.put("$comment", "expression");
}
continue;
}
if (!property.has("type")) {
// inherited properties, such as for expression - supposed to be handled separately
propToRemove.add(propName);
}
}
propToRemove.forEach(processorProperties::remove);
populateDefinitions(processor, relocatedDefinitions);
sanitizeDefinitions(processorFQCN, processor);
answer.put(processorFQCN, processor);
}
return answer;
}

public String getProcessorDefinitionFQCN(String name) {
var processorSchema = getProcessors().withObject("/" + name);
return getNameFromRef(processorSchema);
private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws Exception {
if (!definition.has("oneOf")) {
return definition;
}
var oneOf = definition.withArray("/oneOf");
if (oneOf.size() != 2) {
throw new Exception(String.format(
"Definition '%s' has '%s' entries in oneOf unexpectedly, look it closer",
name,
oneOf.size()));
}
for (var def : oneOf) {
if (def.get("type").asText().equals("object")) {
var objectDef = (ObjectNode) def;
if (definition.has("title")) objectDef.set("title", definition.get("title"));
if (definition.has("description")) objectDef.set("description", definition.get("description"));
return objectDef;
}
}
throw new Exception(String.format(
"Definition '%s' oneOf doesn't have object entry unexpectedly, look it closer",
name));
}

private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) throws Exception {
if (!definition.has("anyOf")) {
return definition;
}
var anyOfOneOf = definition.withArray("/anyOf").get(0).withArray("/oneOf");
for (var def : anyOfOneOf) {
if (def.has("$ref") && def.get("$ref").asText().equals("#/definitions/org.apache.camel.model.language.ExpressionDefinition")) {
definition.put("$comment", "expression");
break;
}
var refParent = def.findParent("$ref");
if (refParent != null && refParent.get("$ref").asText().startsWith("#/definitions/org.apache.camel.model.dataformat")) {
definition.put("$comment", "dataformat");
break;
}
if (LOAD_BALANCE_DEFINITION.equals(name)) {
definition.put("$comment", "loadbalance");
break;
}
}
definition.remove("anyOf");
return definition;
}

private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) throws Exception {
if (!processor.has("definitions")) {
return;
}
var definitions = processor.withObject("/definitions");
var defToRemove = new HashSet<String>();
for (var entry : definitions.properties()) {
var definitionName = entry.getKey();
if (SAGA_DEFINITION.equals(processorFQCN) && definitionName.startsWith("org.apache.camel.language")) {
defToRemove.add(definitionName);
continue;
}

var definition = (ObjectNode) entry.getValue();
definition = extractFromOneOf(definitionName, definition);
definition = extractFromAnyOfOneOf(definitionName, definition);
var definitionProperties = definition.withObject("/properties");
var propToRemove = new HashSet<String>();
for (var property : definitionProperties.properties()) {
var propName = property.getKey();
if (!LOAD_BALANCE_DEFINITION.equals(definitionName) && propName.equals("inheritErrorHandler")) {
// workaround for https://issues.apache.org/jira/browse/CAMEL-20188
// TODO remove this once updated to camel 4.3.0
propToRemove.add(propName);
}
}
propToRemove.forEach(definitionProperties::remove);
if (PROPERTY_EXPRESSION_DEFINITION.equals(definitionName)) {
var expression = definition.withObject("/properties").withObject("/expression");
expression.put("title", "Expression");
expression.put("type", "object");
expression.put("$comment", "expression");
}
definitions.set(definitionName, definition);
}
defToRemove.forEach(definitions::remove);
}
public Map<String, ObjectNode> getDataFormats() throws Exception {
var definitions = yamlDslSchema
.withObject("/items")
Expand Down
12 changes: 11 additions & 1 deletion ...mel-catalog/kaoto-camel-catalog-maven-plugin/src/test/java/CamelCatalogProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,18 @@ public void testGetModelCatalog() throws Exception {
@Test
public void testGetPatternCatalog() throws Exception {
var processorCatalog = jsonMapper.readTree(processor.getPatternCatalog());
assertTrue(processorCatalog.size() > 45 && processorCatalog.size() < 55);
assertTrue(processorCatalog.size() > 55 && processorCatalog.size() < 65);
var choiceModel = processorCatalog.withObject("/choice").withObject("/model");
assertEquals("choice", choiceModel.get("name").asText());
var aggregateSchema = processorCatalog.withObject("/aggregate").withObject("/propertiesSchema");
var aggregationStrategy = aggregateSchema.withObject("/properties").withObject("/aggregationStrategy");
assertEquals("string", aggregationStrategy.get("type").asText());
assertEquals("class:org.apache.camel.AggregationStrategy", aggregationStrategy.get("$comment").asText());

var toDSchema = processorCatalog.withObject("/toD").withObject("/propertiesSchema");
var uri = toDSchema.withObject("/properties").withObject("/uri");
assertEquals("string", uri.get("type").asText());
var parameters = toDSchema.withObject("/properties").withObject("/parameters");
assertEquals("object", parameters.get("type").asText());
}
}
Loading

0 comments on commit e9d71c6

Please sign in to comment.