Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Refactor processors schema #531

Merged
merged 1 commit into from
Dec 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,8 @@
import org.apache.camel.tooling.model.ComponentModel;
import org.apache.camel.tooling.model.EipModel;
import org.apache.camel.tooling.model.JsonMapper;
import org.apache.camel.util.json.JsonObject;

import java.io.StringWriter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
Expand All @@ -38,14 +36,11 @@ public class CamelCatalogProcessor {
private final ObjectMapper jsonMapper;
private final DefaultCamelCatalog api;
private final CamelYamlDslSchemaProcessor schemaProcessor;
private final ArrayList<String> patternBlocklist;

public CamelCatalogProcessor(ObjectMapper jsonMapper, CamelYamlDslSchemaProcessor schemaProcessor) {
this.jsonMapper = jsonMapper;
this.api = new DefaultCamelCatalog();
this.schemaProcessor = schemaProcessor;
patternBlocklist = new ArrayList<>();
populatePatternBlocklist();
}

/**
Expand Down Expand Up @@ -194,16 +189,26 @@ public String getLanguageCatalog() throws Exception {
}

public String getModelCatalog() throws Exception {
var answer = new LinkedHashMap<String, JsonObject>();
var answer = jsonMapper.createObjectNode();
api.findModelNames().stream().sorted().forEach((name) -> {
try {
var model = (EipModel) api.model(Kind.eip, name);
answer.put(name, JsonMapper.asJsonObject(model));
var json = JsonMapper.asJsonObject(model).toJson();
var catalogNode = (ObjectNode) jsonMapper.readTree(json);
if ("from".equals(name)) {
// "from" is an exception that is not a processor, therefore it's not in the
// pattern catalog - put the propertiesSchema here
generatePropertiesSchema(catalogNode);
}
answer.set(name, catalogNode);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
return JsonMapper.serialize(answer);
StringWriter writer = new StringWriter();
var jsonGenerator = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
jsonMapper.writeTree(jsonGenerator, answer);
return writer.toString();
}

/**
Expand All @@ -212,35 +217,44 @@ public String getModelCatalog() throws Exception {
* @throws Exception
*/
public String getPatternCatalog() throws Exception {
var answer = new LinkedHashMap<String, JsonObject>();
var answer = jsonMapper.createObjectNode();
var processors = schemaProcessor.getProcessors();
api.findModelNames().stream().sorted().forEach((name) -> {
var model = (EipModel) api.model(Kind.eip, name);
if (!processors.has(name) || patternBlocklist.contains(name)) {
return;
}
var javaType = schemaProcessor.getProcessorDefinitionFQCN(name);
if (javaType.equals(model.getJavaType())) {
answer.put(name, JsonMapper.asJsonObject(model));
var catalogMap = new LinkedHashMap<String, EipModel>();
for (var name : api.findModelNames()) {
var modelCatalog = (EipModel) api.model(Kind.eip, name);
catalogMap.put(modelCatalog.getJavaType(), modelCatalog);
}
for (var entry : processors.entrySet()) {
var processorFQCN = entry.getKey();
var processorSchema = entry.getValue();
var processorCatalog = catalogMap.get(processorFQCN);
for (var property : processorSchema.withObject("/properties").properties()) {
var propertyName = property.getKey();
var propertySchema = (ObjectNode) property.getValue();
if ("org.apache.camel.model.ToDynamicDefinition".equals(processorFQCN) && "parameters".equals(propertyName)) {
// "parameters" as a common property is omitted in the catalog, but we need this for "toD"
propertySchema.put("title", "Parameters");
propertySchema.put("description", "URI parameters");
continue;
}
var catalogOpOptional = processorCatalog.getOptions().stream().filter(op -> op.getName().equals(propertyName)).findFirst();
if (catalogOpOptional.isEmpty()) {
throw new Exception(String.format("Option '%s' not found for processor '%s'", propertyName, processorFQCN));
}
var catalogOp = catalogOpOptional.get();
if ("object".equals(catalogOp.getType()) && !catalogOp.getJavaType().startsWith("java.util.Map")
&& !propertySchema.has("$comment")) {
propertySchema.put("$comment", "class:" + catalogOp.getJavaType());
}
}
});
return JsonMapper.serialize(answer);
}

private void populatePatternBlocklist() {
this.patternBlocklist.add("kamelet");
this.patternBlocklist.add("loadBalance");
this.patternBlocklist.add("onFallback");
this.patternBlocklist.add("pipeline");
this.patternBlocklist.add("policy");
this.patternBlocklist.add("rollback");
this.patternBlocklist.add("serviceCall");
this.patternBlocklist.add("setExchangePattern");
this.patternBlocklist.add("whenSkipSendToEndpoint");
// reactivate entries once we have a better handling of how to add WHEN and OTHERWISE without Catalog
// this.patternBlocklist.add("Otherwise");
// this.patternBlocklist.add("when");
// this.patternBlocklist.add("doCatch");
// this.patternBlocklist.add("doFinally");
var json = JsonMapper.asJsonObject(processorCatalog).toJson();
var catalogTree = (ObjectNode) jsonMapper.readTree(json);
catalogTree.set("propertiesSchema", processorSchema);
answer.set(processorCatalog.getName(), catalogTree);
}
StringWriter writer = new StringWriter();
var jsonGenerator = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
jsonMapper.writeTree(jsonGenerator, answer);
return writer.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,46 @@
import com.fasterxml.jackson.databind.node.ObjectNode;

import java.io.StringWriter;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Process camelYamlDsl.json file, aka Camel YAML DSL JSON schema.
*/
public class CamelYamlDslSchemaProcessor {
private static final String PROCESSOR_DEFINITION = "org.apache.camel.model.ProcessorDefinition";
private static final String LOAD_BALANCE_DEFINITION = "org.apache.camel.model.LoadBalanceDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION = "org.apache.camel.model.ExpressionSubElementDefinition";
private static final String SAGA_DEFINITION = "org.apache.camel.model.SagaDefinition";
private static final String PROPERTY_EXPRESSION_DEFINITION = "org.apache.camel.model.PropertyExpressionDefinition";
private final ObjectMapper jsonMapper;
private final ObjectNode yamlDslSchema;
private final List<String> processorBlocklist = List.of(
"org.apache.camel.model.KameletDefinition"
// reactivate entries once we have a better handling of how to add WHEN and OTHERWISE without Catalog
// "Otherwise",
// "when",
// "doCatch",
// ""doFinally"
);
/** The processor properties those should be handled separately, i.e. remove from the properties schema,
* such as branching node and parameters reflected from the underlying components. */
private final Map<String, List<String>> processorPropertyBlockList = Map.of(
"org.apache.camel.model.ChoiceDefinition",
List.of("when", "otherwise"),
"org.apache.camel.model.TryDefinition",
List.of("doCatch", "doFinally"),
"org.apache.camel.model.ToDefinition",
List.of("uri", "parameters"),
"org.apache.camel.model.WireTapDefinition",
List.of("uri", "parameters")
);
private final List<String> processorReferenceBlockList = List.of(
PROCESSOR_DEFINITION
);

public CamelYamlDslSchemaProcessor(ObjectMapper mapper, ObjectNode yamlDslSchema) throws Exception {
this.jsonMapper = mapper;
Expand Down Expand Up @@ -96,6 +127,9 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
added = false;
for (JsonNode refParent : schema.findParents("$ref")) {
var name = getNameFromRef((ObjectNode) refParent);
if (processorReferenceBlockList.contains(name)) {
continue;
}
if (!schema.has("definitions") || !schema.withObject("/definitions").has(name)) {
var schemaDefinitions = schema.withObject("/definitions");
schemaDefinitions.set(name, definitions.withObject("/" + name));
Expand All @@ -106,19 +140,171 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
}
}

public ObjectNode getProcessors() {
return yamlDslSchema
/**
* Extract the processor definitions from the main Camel YAML DSL JSON schema in the usable
* format for uniforms to render the configuration form. It does a couple of things:
* <ul>
* <li>Remove "oneOf" and "anyOf"</li>
* <li>Remove properties those are supposed to be handled separately:
* <ul>
* <li>"steps": branching steps</li>
* <li>"parameters": component parameters</li>
* <li>expression languages</li>
* <li>dataformats</li>
* </ul></li>
* <li>If the processor is expression aware, it puts "expression" as a "$comment" in the schema</li>
* <li>If the processor is dataformat aware, it puts "dataformat" as a "$comment" in the schema</li>
* <li>If the processor property is expression aware, it puts "expression" as a "$comment" in the property schema</li>
* </ul>
* @return
*/
public Map<String, ObjectNode> getProcessors() throws Exception {
var definitions = yamlDslSchema
.withObject("/items")
.withObject("/definitions")
.withObject("/org.apache.camel.model.ProcessorDefinition")
.withObject("/definitions");
var relocatedDefinitions = relocateToRootDefinitions(definitions);
var processors = relocatedDefinitions
.withObject(PROCESSOR_DEFINITION)
.withObject("/properties");

var answer = new LinkedHashMap<String, ObjectNode>();
for( var processorEntry : processors) {
var processorFQCN = getNameFromRef((ObjectNode)processorEntry);
if (processorBlocklist.contains(processorFQCN)) {
continue;
}
var processor = relocatedDefinitions.withObject("/" + processorFQCN);
processor = extractFromOneOf(processorFQCN, processor);
processor.remove("oneOf");
processor = extractFromAnyOfOneOf(processorFQCN, processor);
processor.remove("anyOf");
var processorProperties = processor.withObject("/properties");
Set<String> propToRemove = new HashSet<>();
var propertyBlockList = processorPropertyBlockList.get(processorFQCN);
for (var propEntry : processorProperties.properties()) {
var propName = propEntry.getKey();
if (propertyBlockList != null && propertyBlockList.contains(propName)) {
propToRemove.add(propName);
continue;
}
if (!LOAD_BALANCE_DEFINITION.equals(processorFQCN) && propName.equals("inheritErrorHandler")) {
// workaround for https://issues.apache.org/jira/browse/CAMEL-20188
// TODO remove this once updated to camel 4.3.0
propToRemove.add(propName);
continue;
}
var property = (ObjectNode) propEntry.getValue();
var refParent = property.findParent("$ref");
if (refParent != null) {
var ref = getNameFromRef(refParent);
if (processorReferenceBlockList.contains(ref)) {
propToRemove.add(propName);
}
if (EXPRESSION_SUB_ELEMENT_DEFINITION.equals(ref)) {
refParent.remove("$ref");
refParent.put("type", "object");
refParent.put("$comment", "expression");
}
continue;
}
if (!property.has("type")) {
// inherited properties, such as for expression - supposed to be handled separately
propToRemove.add(propName);
}
}
propToRemove.forEach(processorProperties::remove);
populateDefinitions(processor, relocatedDefinitions);
sanitizeDefinitions(processorFQCN, processor);
answer.put(processorFQCN, processor);
}
return answer;
}

public String getProcessorDefinitionFQCN(String name) {
var processorSchema = getProcessors().withObject("/" + name);
return getNameFromRef(processorSchema);
private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws Exception {
if (!definition.has("oneOf")) {
return definition;
}
var oneOf = definition.withArray("/oneOf");
if (oneOf.size() != 2) {
throw new Exception(String.format(
"Definition '%s' has '%s' entries in oneOf unexpectedly, look it closer",
name,
oneOf.size()));
}
for (var def : oneOf) {
if (def.get("type").asText().equals("object")) {
var objectDef = (ObjectNode) def;
if (definition.has("title")) objectDef.set("title", definition.get("title"));
if (definition.has("description")) objectDef.set("description", definition.get("description"));
return objectDef;
}
}
throw new Exception(String.format(
"Definition '%s' oneOf doesn't have object entry unexpectedly, look it closer",
name));
}

private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) throws Exception {
if (!definition.has("anyOf")) {
return definition;
}
var anyOfOneOf = definition.withArray("/anyOf").get(0).withArray("/oneOf");
for (var def : anyOfOneOf) {
if (def.has("$ref") && def.get("$ref").asText().equals("#/definitions/org.apache.camel.model.language.ExpressionDefinition")) {
definition.put("$comment", "expression");
break;
}
var refParent = def.findParent("$ref");
if (refParent != null && refParent.get("$ref").asText().startsWith("#/definitions/org.apache.camel.model.dataformat")) {
definition.put("$comment", "dataformat");
break;
}
if (LOAD_BALANCE_DEFINITION.equals(name)) {
definition.put("$comment", "loadbalance");
break;
}
}
definition.remove("anyOf");
return definition;
}

private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) throws Exception {
if (!processor.has("definitions")) {
return;
}
var definitions = processor.withObject("/definitions");
var defToRemove = new HashSet<String>();
for (var entry : definitions.properties()) {
var definitionName = entry.getKey();
if (SAGA_DEFINITION.equals(processorFQCN) && definitionName.startsWith("org.apache.camel.language")) {
defToRemove.add(definitionName);
continue;
}

var definition = (ObjectNode) entry.getValue();
definition = extractFromOneOf(definitionName, definition);
definition = extractFromAnyOfOneOf(definitionName, definition);
var definitionProperties = definition.withObject("/properties");
var propToRemove = new HashSet<String>();
for (var property : definitionProperties.properties()) {
var propName = property.getKey();
if (!LOAD_BALANCE_DEFINITION.equals(definitionName) && propName.equals("inheritErrorHandler")) {
// workaround for https://issues.apache.org/jira/browse/CAMEL-20188
// TODO remove this once updated to camel 4.3.0
propToRemove.add(propName);
}
}
propToRemove.forEach(definitionProperties::remove);
if (PROPERTY_EXPRESSION_DEFINITION.equals(definitionName)) {
var expression = definition.withObject("/properties").withObject("/expression");
expression.put("title", "Expression");
expression.put("type", "object");
expression.put("$comment", "expression");
}
definitions.set(definitionName, definition);
}
defToRemove.forEach(definitions::remove);
}
public Map<String, ObjectNode> getDataFormats() throws Exception {
var definitions = yamlDslSchema
.withObject("/items")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,18 @@ public void testGetModelCatalog() throws Exception {
@Test
public void testGetPatternCatalog() throws Exception {
var processorCatalog = jsonMapper.readTree(processor.getPatternCatalog());
assertTrue(processorCatalog.size() > 45 && processorCatalog.size() < 55);
assertTrue(processorCatalog.size() > 55 && processorCatalog.size() < 65);
var choiceModel = processorCatalog.withObject("/choice").withObject("/model");
assertEquals("choice", choiceModel.get("name").asText());
var aggregateSchema = processorCatalog.withObject("/aggregate").withObject("/propertiesSchema");
var aggregationStrategy = aggregateSchema.withObject("/properties").withObject("/aggregationStrategy");
assertEquals("string", aggregationStrategy.get("type").asText());
assertEquals("class:org.apache.camel.AggregationStrategy", aggregationStrategy.get("$comment").asText());

var toDSchema = processorCatalog.withObject("/toD").withObject("/propertiesSchema");
var uri = toDSchema.withObject("/properties").withObject("/uri");
assertEquals("string", uri.get("type").asText());
var parameters = toDSchema.withObject("/properties").withObject("/parameters");
assertEquals("object", parameters.get("type").asText());
}
}
Loading
Loading