Release eventgen 7.1.0. Merge to master branch.

Makefile

@@ -39,10 +39,10 @@ test_helper:
 	docker cp . ${EVENTGEN_TEST_IMAGE}:$(shell pwd)
 
 	@echo 'Verifying contents of pip.conf'
-	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); pip install dist/splunk_eventgen*.tar.gz"
+	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); pip3 install dist/splunk_eventgen*.tar.gz"
 
 	@echo 'Installing test requirements'
-	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip install --upgrade pip;pip install -r $(shell pwd)/requirements.txt"
+	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip3 install --upgrade pip;pip3 install -r $(shell pwd)/requirements.txt;pip3 install git+https://github.com/esnme/ultrajson.git"
 
 	@echo 'Make simulated app dir and sample for modular input test'
 	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); cd ../..; mkdir -p modinput_test_app/samples/"
@@ -51,12 +51,15 @@ test_helper:
 	@echo 'Installing docker-compose'
 	bash install_docker_compose.sh
 
+	@echo 'Build a docker image'
+	docker build -t provision_splunk:latest -f tests/large/provision/Dockerfile tests/large/provision
+
 	@echo 'Start container with splunk'
 	docker-compose -f tests/large/provision/docker-compose.yml up &
 
 	sleep 120
 	@echo 'Provision splunk container'
-	docker-compose -f tests/large/provision/docker-compose.yml exec -T splunk sh -c 'cd /opt/splunk;./provision.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart'
+	docker exec --user splunk provision_splunk_1 sh -c 'cd /opt/splunk;./provision.sh;./add_httpevent_collector.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart'
 
 run_tests:
 	@echo 'Running the super awesome tests'

dockerfiles/Dockerfile

@@ -4,6 +4,8 @@ RUN apk --no-cache upgrade && \
 	apk add --no-cache --update \
 	python3 \
 	python3-dev \
+	python2-dev \
+	py2-pip \
 	gcc \
 	libc-dev \
 	libffi-dev \
@@ -26,7 +28,9 @@ RUN apk --no-cache upgrade && \
 	mkdir -p /root/.ssh && \
 	chmod 0700 /root/.ssh && \
 	passwd -u root && \
-	pip3 install git+git://github.com/esnme/ultrajson.git
+	# install dependencies of conduct2 used by perf
+	pip2 install filelock twisted requests queuelib ujson psutil crochet msgpack-python unidecode attrdict service_identity && \
+	pip2 install git+https://github.com/esnme/ultrajson.git
 
 COPY dockerfiles/sshd_config /etc/ssh/sshd_config
 COPY dockerfiles/entrypoint.sh /sbin/entrypoint.sh

docs/CHANGELOG.md

@@ -1,89 +1,58 @@
-6.5.0
-- Added metrics output mode
-- Fixed regex token replacement issue
-- Added test coverage information
-- Increased functional test coverage
-- Eventgen server complete revamp and standalone mode support
-- Added contributor license
-- Updated Dockerfile
-- Added documentation
-- Fixed bugs / stability / optimized speed
-
-6.4.0
-- Fixed exception log error
-- Fixed CircleCI status badage error
-- Fixed navigation error for app if installed with Splunk Stream
-- Fixed generatorWorkers not working error
-- Fixed interval error when end = 1
-- Fixed fileName in global stanza error
-- Added 3rd party libs in SA-Eventgen App
-- Added httpeventAllowFailureCount for httpevent
-- Added 3rd party libs in license credit
-- Disabled logging queue in multiprocess mode
-- Changed implementation of extendIndex for better performance
-
-6.3.6
-- Added functional tests for jinja template and modular input feature
-- Fixed default jinja template directory is not correctly resolved when sampleDir is set issue
-- Fixed verbose flag not working in splunk_eventgen command line issue
-- Fixed index, source, sourcetype are not correct when using splunkstream mode issue
-- Fixed ssh to container issue
-- Fixed perdayvolume without end setting error
-- Updated documentation for better reading and remove unrelated part
-
-6.3.5
-- Added extendIndexes feature to support a list of indexes
-- Fixed timer and token logic
-- Changed end=-1 to continuously iterate without stopping
-- Changed end=0 to not execute
-- Added a linter for code quality
-- Updated docs / docs format
-- Added a suite of functional tests
-
-6.3.4:
-- Documentation cleanup
-- Jinja template bugfix in app
-- Implementation of 'timeMultiple’ option
-- Templates for bugs/feature requests
-- Fixed Jinja test configuration stanzas
-- Default behavior for 'count' edge cases
-
-6.3.3:
-- Added performance metrics compared to Eventgen 5.x
-- New config option for generation-time metrics: outputCounter
-- Jinja template fixes
-- Timestamp parsing fix
-- Output queueing fix for outputMode splunkstream
-- Count rater fixes, now supports indefinite generation
-
-6.3.2:
-- Fixed verbosity bug
-- Added documentation
-
-6.3.1:
-- Fixed Eventgen Volume APIs
-- Improved Eventgen Server Logging
-- Corrected Eventgen Server and Controller conf syncing issue
-- Adding verbosity options (ERROR, INFO, DEBUG) to Eventgen modinput
-- Implemented future event generation support in replay mode
-- Fixed Jinja template's missing default values
-- Adjusted logging message levels for less verbosity
-- Fixed event count off by 1 issue
-- Fixed unnecessary empty data generators being created
-- Updated dependency list
-
-6.3.0:
-- Bug fixes for the customer issues
-- Documentation upgrade
-- Code refactoring for version unification
-- Logging improvements
-
-6.2.1:
-- Fixing SA-Eventgen Dashboard and log searching
-- Improving internal logging and fixing splunkd logging issue
-- Fixing timestamping in default generator
-- Fixing custom plugin integration
-- Fixing SA-Eventgen app settings
-- Supporting Eventgen 5 backward compatibility with additional features
-- Better modinput process management
-- Minor Bugfixes with various customer cases
+**7.1.0**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.1.0)
+
+**7.0.0**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.0.0)
+
+**6.5.2**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.2)
+
+
+**6.5.1**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.1)
+
+
+**6.5.0**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.0)
+
+**6.4.0**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.4.0)
+
+**6.3.6**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.6)
+
+**6.3.5**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.5)
+
+**6.3.4**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.4)
+
+**6.3.3**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.3)
+
+**6.3.2**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.2)
+
+**6.3.1**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.1)
+
+**6.3.0**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.0)
+
+**6.2.1**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.2.1)
+

docs/CONFIGURE.md

@@ -323,8 +323,7 @@ Tokens in the default generator can override the sample to allow dynamic content
       and <end> is a number greater than 0 and greater than or equal to <start>. If rated,
       will be multiplied times hourOfDayRate and dayOfWeekRate.
     * For float[<start>:<end>], the token will be replaced with a random float between
-      start and end values where <start> is a number greater than 0
-      and <end> is a number greater than 0 and greater than or equal to <start>.
+      start and end values where <end> is a number greater than or equal to <start>.
       For floating point numbers, precision will be based off the precision specified
       in <start>. For example, if we specify 1.0, precision will be one digit, if we specify
       1.0000, precision will be four digits. If rated, will be multiplied times hourOfDayRate and dayOfWeekRate.

docs/Gemfile.lock

@@ -206,7 +206,7 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.12.2)
     multipart-post (2.1.1)
-    nokogiri (1.10.4)
+    nokogiri (1.10.8)
       mini_portile2 (~> 2.4.0)
     octokit (4.14.0)
       sawyer (~> 0.8.0, >= 0.5.3)

docs/REFERENCE.md

@@ -121,17 +121,38 @@ outputWorkers = <number of worker threads>
     * Generally if using TCP based outputs like splunkstream, more could be required
     * Defaults to 1
 
-outputMode = modinput | s2s | file | splunkstream | stdout | devnull | spool | httpevent | syslogout | tcpout | udpout | metric_httpevent
+outputMode = scsout | modinput | s2s | file | splunkstream | stdout | devnull | spool | httpevent | syslogout | tcpout | udpout | metric_httpevent
     * Specifies how to output log data. Modinput is default.
+    * If setting scsout, should set scsEndPoint and scsAccessToken. scsClientId, scsClientSecret, and scsRetryNum are optional.
     * If setting spool, should set spoolDir
     * If setting file, should set fileName
     * If setting splunkstream, should set splunkHost, splunkPort, splunkMethod,
       splunkUser and splunkPassword if not Splunk embedded
     * If setting s2s, should set splunkHost and splunkPort
-    * If setting syslogout, should set syslogDestinationHost and syslogDestinationPort
+    * If setting syslogout, should set syslogDestinationHost and syslogDestinationPort. A UDP port listening on Splunk needs to be configured. https://docs.splunk.com/Documentation/Splunk/latest/Data/HowSplunkEnterprisehandlessyslogdata
     * If setting httpevent, should set httpeventServers
     * If setting metric_httpevent, should set httpeventServers and make sure your index is a splunk metric index
 
+scsEndPoint = <host>
+    * Should be a full url to the scs endpoint
+
+scsAccessToken = <token>
+    * Should be a scs access token. Do not include "Bearer". 
+
+scsClientId = <id>
+    * Optional
+    * SCS client id that is used to renew the access token if it expires during the data generation
+    * If not supplied, will not renew the access token and data transmission might fail
+
+scsClientSecret = <secret>
+    * Optional
+    * SCS client secret that is used to renew the access token if it expires during the data generation
+    * If not supplied, will not renew the access token and data transmission might fail
+
+scsRetryNum = <int>
+    * Optional and defaults to 0
+    * Retry a failing data transmission batch
+
 syslogDestinationHost = <host>
     * Defaults to 127.0.0.1
 
@@ -543,8 +564,7 @@ token.<n>.replacement = <string> | <strptime> | ["list","of","strptime"] | guid
       and <end> is a number greater than 0 and greater than or equal to <start>.
       If rated, will be multiplied times hourOfDayRate and dayOfWeekRate.
     * For float[<start>:<end>], the token will be replaced with a random float between
-      start and end values where <start> is a number greater than 0
-      and <end> is a number greater than 0 and greater than or equal to <start>.
+      start and end values where <end> is a number greater than or equal to <start>.
       For floating point numbers, precision will be based off the precision specified
       in <start>. For example, if we specify 1.0, precision will be one digit,
       if we specify 1.0000, precision will be four digits. If rated,

release_tool/README.md

@@ -0,0 +1,15 @@
+# Release tool
+
+Use script to bump the release verison and create the release PR to merge to develop branch.
+
+**Note: this script only works with python3.**
+
+- If you have generated your github access token, you can use the following command to bump versions and send PR automatically.
+    ```bash
+    python prepare_release_branch.py -v -n <release_version> -a <your_access_token>
+    ```
+
+- If the access token is not given, this script only is only used to bump the release version and push the commit to remote repo. You need to go to github web page to create your PR manually.
+    ```
+    python prepare_release_branch.py -v -n <release_version>
+    ```