Skip to content
This repository has been archived by the owner on Dec 9, 2024. It is now read-only.

Commit

Permalink
Merge pull request #105 from igarashitm/202410141051EST
Browse files Browse the repository at this point in the history 
Merge upstream 202410141051 EST
  • Loading branch information
@igarashitm
igarashitm authored Oct 14, 2024
2 parents af6b55c + 6fac494 commit c97b5f0
Show file tree
Hide file tree
Showing 45 changed files with 1,152 additions and 521 deletions.
19 changes: 19 additions & 0 deletions ...atalog-generator/src/main/java/io/kaoto/camelcatalog/generator/CamelCatalogProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,8 @@ public String getDataFormatCatalog() throws Exception {
var json = JsonMapper.asJsonObject(dataFormatCatalog).toJson();
var catalogTree = (ObjectNode) jsonMapper.readTree(json);
catalogTree.set("propertiesSchema", dataFormatSchema);
// setting required property to all the dataformats schema
setRequiredToPropertiesSchema(dataFormatSchema, catalogTree);
answer.set(dataFormatName, catalogTree);
}
StringWriter writer = new StringWriter();
Expand Down Expand Up @@ -267,6 +269,8 @@ public String getLanguageCatalog() throws Exception {
var json = JsonMapper.asJsonObject(languageCatalog).toJson();
var catalogTree = (ObjectNode) jsonMapper.readTree(json);
catalogTree.set("propertiesSchema", languageSchema);
// setting required property to all the languages schema
setRequiredToPropertiesSchema(languageSchema, catalogTree);
answer.set(languageName, catalogTree);
}
StringWriter writer = new StringWriter();
Expand Down Expand Up @@ -709,11 +713,26 @@ public String getLoadBalancerCatalog() throws Exception {
var json = JsonMapper.asJsonObject(loadBalancerCatalog).toJson();
var catalogTree = (ObjectNode) jsonMapper.readTree(json);
catalogTree.set("propertiesSchema", loadBalancerSchema);
// setting required property to all the load-balancers schema
setRequiredToPropertiesSchema(loadBalancerSchema, catalogTree);
answer.set(loadBalancerName, catalogTree);
}
StringWriter writer = new StringWriter();
var jsonGenerator = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
jsonMapper.writeTree(jsonGenerator, answer);
return writer.toString();
}

private void setRequiredToPropertiesSchema(ObjectNode camelYamlDslSchema, ObjectNode catalogModel) {
List<String> required = new ArrayList<>();
var camelYamlDslProperties = camelYamlDslSchema.withObject("/properties").properties().stream()
.map(Map.Entry::getKey).toList();
for (var propertyName : camelYamlDslProperties) {
var catalogPropertySchema = catalogModel.withObject("/properties").withObject("/" + propertyName);
if (catalogPropertySchema.has("required") && catalogPropertySchema.get("required").asBoolean()) {
required.add(propertyName);
}
}
catalogModel.withObject("/propertiesSchema").set("required", jsonMapper.valueToTree(required));
}
}
134 changes: 98 additions & 36 deletions ...-generator/src/main/java/io/kaoto/camelcatalog/generator/CamelYamlDslSchemaProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,42 +15,34 @@
*/
package io.kaoto.camelcatalog.generator;

import java.io.StringWriter;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

import java.io.StringWriter;
import java.util.*;

/**
* Process camelYamlDsl.json file, aka Camel YAML DSL JSON schema.
*/
public class CamelYamlDslSchemaProcessor {
private static final String PROCESSOR_DEFINITION = "org.apache.camel.model.ProcessorDefinition";
private static final String TOKENIZER_DEFINITION = "org.apache.camel.model.TokenizerDefinition";
private static final String ROUTE_CONFIGURATION_DEFINITION = "org.apache.camel.model.RouteConfigurationDefinition";
private static final String LOAD_BALANCE_DEFINITION = "org.apache.camel.model.LoadBalanceDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION = "org.apache.camel.model.ExpressionSubElementDefinition";
private static final String EXPRESSION_SUB_ELEMENT_DEFINITION =
"org.apache.camel.model.ExpressionSubElementDefinition";
private static final String SAGA_DEFINITION = "org.apache.camel.model.SagaDefinition";
private static final String PROPERTY_EXPRESSION_DEFINITION = "org.apache.camel.model.PropertyExpressionDefinition";
private static final String ERROR_HANDLER_DEFINITION = "org.apache.camel.model.ErrorHandlerDefinition";
private static final String ERROR_HANDLER_DESERIALIZER = "org.apache.camel.dsl.yaml.deserializers.ErrorHandlerBuilderDeserializer";
private static final String ERROR_HANDLER_DESERIALIZER =
"org.apache.camel.dsl.yaml.deserializers.ErrorHandlerBuilderDeserializer";
private final ObjectMapper jsonMapper;
private final ObjectNode yamlDslSchema;
private final List<String> processorBlocklist = List.of(
"org.apache.camel.model.KameletDefinition"
// reactivate entries once we have a better handling of how to add WHEN and
// OTHERWISE without Catalog
// "Otherwise",
// "when",
// "doCatch",
// ""doFinally"
);
private final List<String> processorBlocklist = List.of("org.apache.camel.model.KameletDefinition");

/**
* The processor properties those should be handled separately, i.e. remove from
Expand All @@ -67,8 +59,7 @@ public class CamelYamlDslSchemaProcessor {
List.of("uri", "parameters"),
"org.apache.camel.model.WireTapDefinition",
List.of("uri", "parameters"));
private final List<String> processorReferenceBlockList = List.of(
PROCESSOR_DEFINITION);
private final List<String> processorReferenceBlockList = List.of(PROCESSOR_DEFINITION);

public CamelYamlDslSchemaProcessor(ObjectMapper mapper, ObjectNode yamlDslSchema) throws Exception {
this.jsonMapper = mapper;
Expand Down Expand Up @@ -107,11 +98,14 @@ private String doProcessSubSchema(
var answer = (ObjectNode) prop.getValue().deepCopy();
if (answer.has("$ref") && definitions.has(getNameFromRef(answer))) {
answer = definitions.withObject("/" + getNameFromRef(answer)).deepCopy();

}

extractSingleOneOfFromAnyOf(answer);
removeEmptyProperties(answer);
removeNotFromOneOf(answer);
answer.set("$schema", rootSchema.get("$schema"));
populateDefinitions(answer, definitions);

var writer = new StringWriter();
try {
JsonGenerator gen = new JsonFactory().createGenerator(writer).useDefaultPrettyPrinter();
Expand All @@ -122,6 +116,58 @@ private String doProcessSubSchema(
}
}

/**
* Remove the empty properties from the definition, as it's not supported by the
* form library. This happens when the properties are supposed to be handled
* separately, in a oneOf definition, for example.
* An example:
* ```
* {
* properties: {
* deadLetterChannel: { },
* defaultErrorHandler: { }
* }
* }
*
* @param definitionWithEmptyProperties the definition containing the empty properties to be removed
*/
private void removeEmptyProperties(ObjectNode definitionWithEmptyProperties) {
if (!definitionWithEmptyProperties.has("properties")) {
return;
}

var propertiesObject = definitionWithEmptyProperties.withObject("/properties");
List<String> propToRemove = new ArrayList<>();
propertiesObject.fields().forEachRemaining(field -> {
if (field.getValue().isEmpty()) {
propToRemove.add(field.getKey());
}
});
propToRemove.forEach(propertiesObject::remove);
}

/**
* Remove "not" from the OneOf definition, as it's not supported by the
* form library. The "not" is only useful for the Source code editor.
*
* @param definitionContainingOneOf the definition to be cleaned
*/
private void removeNotFromOneOf(ObjectNode definitionContainingOneOf) {
if (!definitionContainingOneOf.has("oneOf")) {
return;
}

ArrayNode cleanAnyOf = jsonMapper.createArrayNode();
var oneOfDefinitionArray = definitionContainingOneOf.withArray("/oneOf");
for (var def : oneOfDefinitionArray) {
if (def.has("not")) {
continue;
}
cleanAnyOf.add(def);
}
definitionContainingOneOf.set("oneOf", cleanAnyOf);
}

private String getNameFromRef(ObjectNode parent) {
var ref = parent.get("$ref").asText();
return ref.contains("items") ? ref.replace("#/items/definitions/", "")
Expand Down Expand Up @@ -152,9 +198,7 @@ private void populateDefinitions(ObjectNode schema, ObjectNode definitions) {
* root definitions.
* It's a workaround for the current Camel YAML DSL JSON schema, where some
* AnyOf definition
* contains only one OneOf definition. This can be removed once
* https://github.com/KaotoIO/kaoto/issues/948
* is resolved.
* contains only one OneOf definition.
* This is done mostly for the errorHandler definition, f.i.
* ```
* {
Expand Down Expand Up @@ -214,7 +258,7 @@ private void extractSingleOneOfFromAnyOf(ObjectNode definition) {
* "$comment" in the property schema</li>
* </ul>
*
* @return
* @return A map of processor definitions
*/
public Map<String, ObjectNode> getProcessors() throws Exception {
var definitions = yamlDslSchema
Expand All @@ -235,8 +279,16 @@ public Map<String, ObjectNode> getProcessors() throws Exception {
var processor = relocatedDefinitions.withObject("/" + processorFQCN);
processor = extractFromOneOf(processorFQCN, processor);
processor.remove("oneOf");
processor = extractFromAnyOfOneOf(processorFQCN, processor);
processor.remove("anyOf");

/* Preparation for TokenizerDefinition, this could be propagated to all EIPs in the future */
if (processorFQCN.equals(TOKENIZER_DEFINITION)) {
removeEmptyProperties(processor);
extractSingleOneOfFromAnyOf(processor);
removeNotFromOneOf(processor);
}

processAndRemoveAnyOfForSubCatalogs(processorFQCN, processor);

var processorProperties = processor.withObject("/properties");
Set<String> propToRemove = new HashSet<>();
var propertyBlockList = processorPropertyBlockList.get(processorFQCN);
Expand Down Expand Up @@ -316,10 +368,12 @@ private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws E
for (var def : oneOf) {
if (def.get("type").asText().equals("object")) {
var objectDef = (ObjectNode) def;
if (definition.has("title"))
if (definition.has("title")) {
objectDef.set("title", definition.get("title"));
if (definition.has("description"))
}
if (definition.has("description")) {
objectDef.set("description", definition.get("description"));
}
return objectDef;
}
}
Expand All @@ -328,9 +382,19 @@ private ObjectNode extractFromOneOf(String name, ObjectNode definition) throws E
name));
}

private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) throws Exception {
/**
* Process the "anyOf" definition for the sub-catalogs, such as expressions, languages,
* data formats, and errorHandler.
* It puts a "$comment" in the schema to indicate the type of the sub-catalog and then removes
* the "anyOf" definition.
*
* @param name the FQCN of the definition, for instance "org.apache.camel.model.LoadBalanceDefinition"
* @param definition the definition that potentially could have "anyOf" definition, referencing to another
* sub-catalogs
*/
private void processAndRemoveAnyOfForSubCatalogs(String name, ObjectNode definition) {
if (!definition.has("anyOf")) {
return definition;
return;
}
var anyOfOneOf = definition.withArray("/anyOf").get(0).withArray("/oneOf");
for (var def : anyOfOneOf) {
Expand All @@ -355,7 +419,6 @@ private ObjectNode extractFromAnyOfOneOf(String name, ObjectNode definition) thr
}
}
definition.remove("anyOf");
return definition;
}

private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) throws Exception {
Expand All @@ -373,7 +436,7 @@ private void sanitizeDefinitions(String processorFQCN, ObjectNode processor) thr

var definition = (ObjectNode) entry.getValue();
definition = extractFromOneOf(definitionName, definition);
definition = extractFromAnyOfOneOf(definitionName, definition);
processAndRemoveAnyOfForSubCatalogs(definitionName, definition);
var definitionProperties = definition.withObject("/properties");
var propToRemove = new HashSet<String>();
for (var property : definitionProperties.properties()) {
Expand Down Expand Up @@ -529,7 +592,7 @@ public Map<String, ObjectNode> getLanguages() throws Exception {
* <li>If the processor property is expression aware, it puts "expression" as a
* "$comment" in the property schema</li>
*
* @return
* @return A map of the entity name and the schema
*/
public Map<String, ObjectNode> getEntities() throws Exception {
var definitions = yamlDslSchema
Expand All @@ -548,8 +611,7 @@ public Map<String, ObjectNode> getEntities() throws Exception {
var yamlInDefinition = relocatedDefinitions.withObject("/" + yamlInFQCN);
yamlInDefinition = extractFromOneOf(yamlInFQCN, yamlInDefinition);
yamlInDefinition.remove("oneOf");
yamlInDefinition = extractFromAnyOfOneOf(yamlInFQCN, yamlInDefinition);
yamlInDefinition.remove("anyOf");
processAndRemoveAnyOfForSubCatalogs(yamlInFQCN, yamlInDefinition);
Set<String> propToRemove = new HashSet<>();
var yamlInProperties = yamlInDefinition.withObject("/properties");
for (var yamlInPropertyEntry : yamlInProperties.properties()) {
Expand Down
5 changes: 1 addition & 4 deletions packages/catalog-generator/src/main/java/io/kaoto/camelcatalog/generator/Util.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.commons.io.FilenameUtils;

public class Util {
public static String generateHash(byte[] content) throws Exception {
if (content == null)
Expand All @@ -44,8 +42,7 @@ public static String getNormalizedFolder(String folder) {

// Resolve the relative path
Path absolutePath = currentDirectory.resolve(folder);
String normalizedfolder = FilenameUtils.separatorsToUnix(absolutePath.toString());

return normalizedfolder;
return absolutePath.toString();
}
}
2 changes: 1 addition & 1 deletion ...og-generator/src/test/java/io/kaoto/camelcatalog/commands/GenerateCommandOptionsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public void testConfigureWithAllRequiredOptions() throws ParseException {
generateCommandOptions.configure(args);
String outputDir = Util.getNormalizedFolder("outputDir");

assertEquals(outputDir, configBean.getOutputFolder().toString());
assertEquals(outputDir, configBean.getOutputFolder().toPath().toString());
assertEquals("catalogName", configBean.getCatalogsName());
assertEquals("kameletsVersion", configBean.getKameletsVersion());
}
Expand Down
10 changes: 7 additions & 3 deletions ...s/catalog-generator/src/test/java/io/kaoto/camelcatalog/commands/GenerateCommandTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.junit.jupiter.api.io.TempDir;

import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;

import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -84,7 +85,8 @@ void testGeneratorCalledWithCorrectParameters() {
File expectedFolder = new File(tempDir, "camel-main/4.8.0");
verify(builder, times(1)).withOutputDirectory(expectedFolder);

assertEquals(catalogDefinition.getFileName(), "camel-main/4.8.0/index.json");
String expectedFile = Path.of("camel-main", "4.8.0", "index.json").toString();
assertEquals(catalogDefinition.getFileName(), expectedFile);
}
}

Expand Down Expand Up @@ -123,7 +125,9 @@ void testCatalogLibraryOutput() {
assertEquals(catalogLibraryEntry.name(), "test-camel-catalog");
assertEquals(catalogLibraryEntry.version(), "4.8.0");
assertEquals(catalogLibraryEntry.runtime(), "Main");
assertEquals(catalogLibraryEntry.fileName(), "camel-main/4.8.0/index.json");

String expectedFile = Path.of("camel-main", "4.8.0", "index.json").toString();
assertEquals(catalogLibraryEntry.fileName(), expectedFile);
}
}
}
}
Loading

0 comments on commit c97b5f0

Please sign in to comment.