Skip to content

Commit

Permalink
#332 Add Test And Documentation
Browse files Browse the repository at this point in the history
Removed some unused test files
  • Loading branch information
AdrianLagartera committed Mar 1, 2023
1 parent 9a197c7 commit af0dfe3
Show file tree
Hide file tree
Showing 7 changed files with 212 additions and 287 deletions.
20 changes: 20 additions & 0 deletions docs/schemas.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,3 +176,23 @@ The same applies when there is more than one level. The last child in the last l
The other exception applies to JSON Schema: in order to support `null`values, the collections within objects (type : object) cannot be null. Therefore, they will be **required by default** and they will be **initialized as an empty collection** if the object than contains them is not null.

> Within JSON Schema, a maximum of **2 nesting levels** is allowed.

### Protobuf Schema - Schema Registry

If you need use the Protobuf Schema with the Schema Registry, you must put the subject with the same name as the protobuf file, as the following example:

```
{
"schema": "...",
"schemaType": "PROTOBUF",
"references": [
{
"name": "[the_name_you_want].proto",
"subject": "[the_name_you_want]",
"version": ...
}]
}
```
> This example is based on a petition from Schema Registry
2 changes: 1 addition & 1 deletion pom-maven-central.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

<artifactId>kloadgen</artifactId>

<version>5.3.1</version>
<version>5.3.2</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data based on Data specification</description>
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

<artifactId>kloadgen</artifactId>

<version>5.3.1</version>
<version>5.3.2</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification</description>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* * License, v. 2.0. If a copy of the MPL was not distributed with this
* * file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/

package com.sngular.kloadgen.loadgen.impl;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
import com.sngular.kloadgen.exception.KLoadGenException;
import com.sngular.kloadgen.model.FieldValueMapping;
import com.sngular.kloadgen.serializer.EnrichedRecord;
import com.sngular.kloadgen.util.ProducerKeysHelper;
import com.sngular.kloadgen.util.SchemaRegistryKeyHelper;
import io.confluent.kafka.schemaregistry.client.SchemaRegistryClientConfig;
import org.apache.jmeter.threads.JMeterContext;
import org.apache.jmeter.threads.JMeterContextService;
import org.apache.jmeter.threads.JMeterVariables;
import org.apache.jmeter.util.JMeterUtils;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@WireMockTest
class ProtobufSRLoadGeneratorTest {

@BeforeEach
public void setUp(final WireMockRuntimeInfo wmRuntimeInfo) throws IOException {
final File file = new File("src/test/resources");
final String absolutePath = file.getAbsolutePath();
final String kloadPath = absolutePath + "/kloadgen.properties";
final Map<String, String> properties = new HashMap<>();
properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl());
properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_KEY, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_BASIC_TYPE);
properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_FLAG, ProducerKeysHelper.FLAG_YES);
properties.put(SchemaRegistryClientConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "foo");
properties.put(SchemaRegistryClientConfig.USER_INFO_CONFIG,"foo");
JMeterUtils.loadJMeterProperties(kloadPath);
final JMeterContext jmcx = JMeterContextService.getContext();
jmcx.getProperties().putAll(properties);
jmcx.setVariables(new JMeterVariables());
JMeterUtils.setLocale(Locale.ENGLISH);
}

@Test
void testAvroLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws KLoadGenException {

final List<FieldValueMapping> fieldValueMappingList = Arrays.asList(
FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true)
.isAncestorRequired(true).build(),
FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true)
.isAncestorRequired(true).build(),
FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true)
.isAncestorRequired(true).build(),
FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true)
.isAncestorRequired(true).build()
);

final Map<String, String> originals = new HashMap<>();
originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl());
originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY, "foo");
originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY, "foo");

final ProtobufLoadGenerator protobufLoadGenerator = new ProtobufLoadGenerator();
protobufLoadGenerator.setUpGenerator(originals, "protobufSubjectWithImport", fieldValueMappingList);
final Object message = protobufLoadGenerator.nextMessage();
Assertions.assertThat(message).isNotNull().isInstanceOf(EnrichedRecord.class);

final EnrichedRecord enrichedRecord = (EnrichedRecord) message;
Assertions.assertThat(enrichedRecord.getGenericRecord()).isNotNull();
Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyTest1");
Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("entityNumberTwo");
Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberOne");
Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberTwo");
}
}
106 changes: 105 additions & 1 deletion src/test/resources/mappings/schema_registry_stub.json
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,86 @@
}
}
},
{
"priority": 10,
"request": {
"method": "GET",
"urlPattern": "/subjects/protobuf_subject/versions/latest"
},
"response": {
"status": 200,
"jsonBody": {
"subject": "ProtobufSubject",
"version": 1,
"id": 74,
"schemaType": "PROTOBUF",
"schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\noption java_package = \"com.sngular.external.proto\";\r\noption java_outer_classname = \"ExternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage ExternalEntity{\r\n string parentPropertyOne = 1;\r\n int32 parentPropertyTwo = 2;\r\n \r\n message EntityOne {\r\n int32 entityNumberOne = 1;\r\n string entityNumberTwo = 2;\r\n \r\n message NestingEntity {\r\n string nestedProperty = 1;\r\n \r\n }\r\n \r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}"
}
}
},
{
"priority": 10,
"request": {
"method": "GET",
"urlPattern": "/schemas/ids/74\\?fetchMaxId=false&subject=protobuf_subject"
},
"response": {
"status": 200,
"jsonBody": {
"schemaType": "PROTOBUF",
"schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\noption java_package = \"com.sngular.external.proto\";\r\noption java_outer_classname = \"ExternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage ExternalEntity{\r\n string parentPropertyOne = 1;\r\n int32 parentPropertyTwo = 2;\r\n \r\n message EntityOne {\r\n int32 entityNumberOne = 1;\r\n string entityNumberTwo = 2;\r\n \r\n message NestingEntity {\r\n string nestedProperty = 1;\r\n \r\n }\r\n \r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}"
},
"headers": {
"Content-Type": "application/vnd.schemaregistry.v1+json"
}
}
},
{
"priority": 10,
"request": {
"method": "GET",
"urlPattern": "/subjects/protobufSubjectWithImport/versions/latest"
},
"response": {
"status": 200,
"jsonBody": {
"subject": "ProtobufSubjectWithImport",
"version": 1,
"id": 75,
"schemaType": "PROTOBUF",
"schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\nimport \"protobuf_subject.proto\";\r\n\r\noption java_package = \"com.sngular.internal.proto\";\r\noption java_outer_classname = \"InternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage InternalMessage{\r\n\r\n InternalEntityOne propertyTest1 = 1;\r\n EntityTwo propertyTest2 = 2;\r\n \r\n message InternalEntityOne {\r\n ExternalEntity.EntityOne.NestingEntity importedProperty = 1;\r\n string entityNumberTwo = 2;\r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}",
"references" : [{
"name": "protobuf_subject.proto",
"subject": "protobuf_subject",
"version": 1
}
]
}
}
},
{
"priority": 10,
"request": {
"method": "GET",
"urlPattern": "/schemas/ids/75\\?fetchMaxId=false&subject=protobufSubjectWithImport"
},
"response": {
"status": 200,
"jsonBody": {
"schemaType": "PROTOBUF",
"schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\nimport \"protobuf_subject.proto\";\r\n\r\noption java_package = \"com.sngular.internal.proto\";\r\noption java_outer_classname = \"InternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage InternalMessage{\r\n\r\n InternalEntityOne propertyTest1 = 1;\r\n EntityTwo propertyTest2 = 2;\r\n \r\n message InternalEntityOne {\r\n ExternalEntity.EntityOne.NestingEntity importedProperty = 1;\r\n string entityNumberTwo = 2;\r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}",
"references" : [{
"name": "protobuf_subject.proto",
"subject": "protobuf_subject",
"version": 1
}
]
},
"headers": {
"Content-Type": "application/vnd.schemaregistry.v1+json"
}
}
},
{
"priority": 10,
"request": {
Expand All @@ -119,12 +199,36 @@
"jsonBody": [
"avrosubject",
"users",
"arrayMap"
"arrayMap",
"protobuf_subject",
"protobufSubjectWithImport"
],
"headers": {
"Content-Type": "application/vnd.schemaregistry.v1+json"
}
}
},
{
"priority": 10,
"request": {
"method": "GET",
"urlPattern" : "/subjects/protobuf_subject/versions/1\\?deleted=true"
},
"response": {
"status": 200,
"jsonBody": {
"subject": "protobuf_subject",
"version": 1,
"id": 74,
"schemaType": "PROTOBUF",
"schema": "syntax = \"proto3\";\npackage sngular;\n\noption java_package = \"com.sngular.external.proto\";\noption java_outer_classname = \"ExternalMessage\";\noption java_multiple_files = true;\n\nmessage ExternalEntity{\n string parentPropertyOne = 1;\n int32 parentPropertyTwo = 2;\n \n message EntityOne {\n int32 entityNumberOne = 1;\n string entityNumberTwo = 2;\n \n message NestingEntity {\n string nestedProperty = 1;\n \n }\n \n }\n \n message EntityTwo {\n int32 propertyNumberOne = 1;\n string propertyNumberTwo = 2; \n }\n\n}"
},
"headers": {
"Content-Type": "application/vnd.schemaregistry.v1+json"
}
}


}
]
}
Loading

0 comments on commit af0dfe3

Please sign in to comment.