Merge pull request #405 from splunk/develop

Release 7.1.1

.circleci/config.yml

@@ -6,16 +6,30 @@ version: 2
 jobs:
   test_eventgen:
     machine:
-      image: circleci/classic:latest
+      image: circleci/classic:201808-01
     steps:
       - checkout
       - run:
-          name: Install Dependencies
+          name: Enable Py3
           command: |
-            pip install --upgrade pip && pip install -r requirements.txt
+            pyenv global 3.7.0
+      - run:
+          name: Run Flake8 Code Lint
+          command: |
+            set -e
+            pip install "flake8>=3.7.7"
+            make lint-all
+      - run:
+          name: Check Code Format
+          command: |
+            set -e
+            pip install "black==19.10b0" "isort>=4.3.15"
+            make format-check
       - run:
           name: Run Tests
           command: |
+            pip3 install --upgrade pip
+            pip3 install poetry
             set -e
             make test
           no_output_timeout: 30m
@@ -31,4 +45,3 @@ workflows:
   run_tests:
     jobs:
       - test_eventgen
-

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+exclude = .git,.tox,__pycache__,env,venv,build
+max-line-length = 120
+extend-ignore = E203,E121,E123,E126,E226,E24,E704,W503,W504,E722,E731,W605
+# Includes default ignores, E722 (bare excepts), E731 (lambda usage), and W605 (escape sequences)

Makefile

@@ -12,13 +12,14 @@ LARGE ?= 'tests/large'
 XLARGE ?= 'tests/xlarge'
 NEWLY_ADDED_PY_FILES = $(shell git ls-files -o --exclude-standard | grep -E '\.py$$')
 CHANGED_ADDED_PY_FILES = $(shell git ls-files -mo --exclude-standard | grep -E '\.py$$')
+ALL_PY_FILES = $(shell git ls-files tests splunk_eventgen | grep -E '\.py$$')
 
 .PHONY: tests, lint, format, docs
 
 all: egg
 
 egg: clean
-	python setup.py sdist
+	poetry build
 
 image: setup_eventgen egg
 	rm splunk_eventgen/default/eventgen_engine.conf || true
@@ -41,15 +42,15 @@ test_helper:
 	@echo 'Verifying contents of pip.conf'
 	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); pip3 install dist/splunk_eventgen*.tar.gz"
 
-	@echo 'Installing test requirements'
-	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip3 install --upgrade pip;pip3 install -r $(shell pwd)/requirements.txt;pip3 install git+https://github.com/esnme/ultrajson.git"
+	@echo 'Installing test dependencies'
+	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip3 install poetry;poetry config virtualenvs.create false;poetry install --no-root"
 
 	@echo 'Make simulated app dir and sample for modular input test'
 	docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); cd ../..; mkdir -p modinput_test_app/samples/"
 	docker cp tests/large/sample/film.json ${EVENTGEN_TEST_IMAGE}:$(shell pwd)/../../modinput_test_app/samples
 
 	@echo 'Installing docker-compose'
-	bash install_docker_compose.sh
+	bash tests/large/provision/install_docker_compose.sh
 
 	@echo 'Build a docker image'
 	docker build -t provision_splunk:latest -f tests/large/provision/Dockerfile tests/large/provision
@@ -123,14 +124,15 @@ run_standalone:
 	docker run --name eg_standalone  -d -p 9500:9500 eventgen:latest standalone
 
 run_local_standalone:
-	python -m splunk_eventgen service -r standalone
+	python3 -m splunk_eventgen service -r standalone
 
 docs:
 	cd docs/; bundle install; bundle exec jekyll serve
 
 build_spl: clean
 	python3 -m splunk_eventgen build --destination ./
 
+
 lint:
 ifeq ($(NEWLY_ADDED_PY_FILES), )
 	@echo 'No newly added python files. Skip...'
@@ -148,12 +150,18 @@ endif
 ifeq ($(NEWLY_ADDED_PY_FILES), )
 	@echo 'No newly added python files. Skip...'
 else
-	@yapf -i $(NEWLY_ADDED_PY_FILES)
+	@black -t py37 $(NEWLY_ADDED_PY_FILES)
 endif
 
 lint-all:
-	@flake8 .
+	@echo "lint all py files"
+	@flake8 $(ALL_PY_FILES)
+
+format-check:
+	@echo 'Checking all py files code format'
+	@black --check -t py37 .
 
 format-all:
-	@isort -rc .
-	@yapf -r -i .
+	@echo "format all py files"
+	@isort -rc $(ALL_PY_FILES)
+	@black -t py37 $(ALL_PY_FILES)

dockerfiles/Dockerfile

@@ -28,15 +28,17 @@ RUN apk --no-cache upgrade && \
 	mkdir -p /root/.ssh && \
 	chmod 0700 /root/.ssh && \
 	passwd -u root && \
-	# install dependencies of conduct2 used by perf
-	pip2 install filelock twisted requests queuelib ujson psutil crochet msgpack-python unidecode attrdict service_identity && \
-	pip2 install git+https://github.com/esnme/ultrajson.git
+	# install dependencies of conductor2 used by perf
+	pip2 install filelock twisted requests queuelib psutil crochet msgpack-python unidecode attrdict service_identity && \
+	pip2 install git+https://github.com/esnme/ultrajson.git@v1.35
 
 COPY dockerfiles/sshd_config /etc/ssh/sshd_config
 COPY dockerfiles/entrypoint.sh /sbin/entrypoint.sh
-COPY dist/* /root/splunk_eventgen.tgz
+COPY dist/splunk_eventgen*.tar.gz /root/splunk_eventgen.tgz
 RUN pip3 install /root/splunk_eventgen.tgz && \
 	rm /root/splunk_eventgen.tgz
+COPY pyproject.toml /usr/lib/python3.7/site-packages/splunk_eventgen/pyproject.toml
+COPY poetry.lock /usr/lib/python3.7/site-packages/splunk_eventgen/poetry.lock
 
 EXPOSE 2222 6379 9500
 RUN chmod a+x /sbin/entrypoint.sh

docs/CHANGELOG.md

@@ -1,3 +1,7 @@
+**7.1.1**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.1.1)
+
 **7.1.0**:
 
 - Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.1.0)

docs/CONFIGURE.md

@@ -374,7 +374,7 @@ specifically be supported by all plugins. Plugins that write to files like spool
 
     source = <source>
     * Valid with outputMode=modinput (default) & outputMode=splunkstream & outputMode=httpevent
-    * Set event source in Splunk to <source>. Defaults to 'eventgen' if none specified.
+    * Set event source in Splunk to <source>. Defaults to sample file name if none specified.
 
     sourcetype = <sourcetype>
     * Valid with outputMode=modinput (default) & outputMode=splunkstream & outputMode=httpevent

docs/REFERENCE.md

@@ -44,7 +44,6 @@ fileBackupFiles = 5
 splunkPort = 8089
 splunkMethod = https
 index = main
-source = eventgen
 sourcetype = eventgen
 host = 127.0.0.1
 outputWorkers = 1
@@ -265,7 +264,7 @@ extendIndexes = <index_prefix>:<weight>,<index2>,<index3>
 source = <source>
     * Valid with the following outputMode:
       outputMode=modinput (default) & outputMode=splunkstream & outputMode=httpevent
-    * Set event source in Splunk to <source>. Defaults to 'eventgen' if none specified.
+    * Set event source in Splunk to <source>. Defaults to sample file name if none specified.
 
 sourcetype = <sourcetype>
     * Valid with the following outputMode:

docs/SETUP.md

@@ -1,47 +1,64 @@
 ## Install
 
+For upgrading eventgen, please refer to [this doc](UPGRADE.md).
+
 There are multiple ways to use Eventgen, and you should choose the method that best fits your use case.
 Below are the two major ways to use Eventgen - as a PyPI module and as a Splunk App. Follow the instructions below depending on your ideal use:
 
-* Install / Use Eventgen as a [Splunk App](#splunk-app-installation)
+1. Install / Use Eventgen as a [Splunk App](#splunk-app-installation)
+
+    > :heavy_exclamation_mark: Starting from 7.0.0 release, Eventgen only supports python3, which means SA-eventgen can only work with Splunk enterprise 8.x and later release.
 
     Benefits:
-  * Easy To Install
-  * Works with TA's downloaded direclty from SplunkBase
-  * Uses a modular input for controlling the main Eventgen process
-  * Reads configurations out of Splunk Rest
-  * Supports apps default / local directories
-
-  Draw Backs:
-  * Limited to a single process
-  * Can't scale to large datasets
-  * Easily can fall behind on processing large quantities of eventgen.conf files
-  * Doesn't install the jinja templating system automatically
-  * No support for python multi-threading / processing
-
-* Install / Use Eventgen as a [Python (PyPI) package](#pypi-installation)
-
-  Benefits:
-  * Support for threading / multiprocessing
-  * Support for a centralized service that can controll and run multiple threading workers
-  * Able to run a larger amount of datavolume with less overhead
-  * Allows an Eventgen object to be embeded and controlled using python code
-  * Exposes more of the plugin system
-  * Includes/installs the Jinja2 templating engine
-
-  Drawbacks:
-  * More complex installation
-  * You have to run the `build` command to produce a Splunk app
-  * Harder to troubleshoot (especially in multiprocess mode)
+    * Easy To Install
+    * Works with TA's downloaded direclty from SplunkBase
+    * Uses a modular input for controlling the main Eventgen process
+    * Reads configurations out of Splunk Rest
+    * Supports apps default / local directories
+
+    Draw Backs:
+    * Limited to a single process
+    * Can't scale to large datasets
+    * Easily can fall behind on processing large quantities of eventgen.conf files
+    * Doesn't install the jinja templating system automatically
+    * No support for python multi-threading / processing
+
+
+1. Install / Use Eventgen as a [Python (PyPI) package](#pypi-installation)
+
+    Benefits:
+    * Support for threading / multiprocessing
+    * Support for a centralized service that can controll and run multiple threading workers
+    * Able to run a larger amount of datavolume with less overhead
+    * Allows an Eventgen object to be embeded and controlled using python code
+    * Exposes more of the plugin system
+    * Includes/installs the Jinja2 templating engine
+
+    Drawbacks:
+    * More complex installation
+    * You have to run the `build` command to produce a Splunk app
+    * Harder to troubleshoot (especially in multiprocess mode)
 
 ---
 
 <a id="pypi-installation"></a>
-## PyPI Installation / First Run
+### PyPI Installation / First Run
+
+> :heavy_exclamation_mark: Starting from 7.0.0 release, Eventgen only supports python3.
+
+To use Eventgen as a PyPI module, you need to either download/clone the source code or install direct from github.
+
+#### Prerequisite
+
+1. Install git.
+1. Install python3 before building Eventgen>=7.0.0.
+1. Install python2 before building Eventgen<7.0.0.
 
-To use Eventgen as a PyPI module, you need to either download/clone the source code or install direct from github. 
+>:information_source: You can download the python3 installation package [here](https://www.python.org/downloads/).
 
-###### Download Sourcecode
+#### PiPI Installation
+
+##### Download Eventgen Sourcecode
 ```
 $ git clone https://www.github.com/splunk/eventgen
 ```
@@ -54,37 +71,48 @@ $ git branch -a
   remotes/origin/HEAD -> origin/develop
   remotes/origin/develop
   remotes/origin/master
-  
+
 $ git checkout remotes/origin/master
 Note: checking out 'remotes/origin/master'.
 
 $ git pull
 ```
+Eventgen uses [Poetry](https://github.com/python-poetry/poetry) to manage and build. Install `Poetry` first:
+```
+curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+
+```
+Or you can use `pip` to install `poetry`(this is not recommended which might cause conflicts):
+```
+pip install --user poetry
+```
 
 Then run below commands inside Eventgen directory:
 ```
-$ python setup.py sdist
+$ poetry build
 
 # you should see a tar file inside of dist directory
+# the <version> is a token to represent current version of Eventgen, for example, 7.1.0
 $ ls dist
-splunk_eventgen-6.x.x.tar.gz
+splunk_eventgen-<version>.tar.gz
 
-$ pip install splunk_eventgen-6.x.x.tar.gz
+$ pip install splunk_eventgen-<version>.tar.gz
 
 ```
-###### Install Direct From GitHub
+##### Install Direct From GitHub
 To install Eventgen direct from github, use the following pip syntax:
 
 ```
 $ pip install git+https://www.github.com/splunk/eventgen.git
 ```
 
-###### Verify Installation
+##### Verify Installation
 
 After completing either of the above install methods, you can verify seccussful installation by checking the packaged Eventgen version.
 ```
+# the <version> is a token to represent current version of Eventgen, for example, 7.1.0
 $ splunk_eventgen --version
-Eventgen 6.x.x
+Eventgen <version>
 ```
 Now you are ready to use Eventgen as a pip module.
 
@@ -115,13 +143,13 @@ Please follow these instructions to run an Eventgen cluster on your Docker envir
 ---
 
 <a id="splunk-app-installation"></a>
-## Splunk App Installation / First Run
+### Splunk App Installation / First Run
 
 To use Eventgen as a Splunk app, you need a SPL file. This SPL file can be obtained in one of two ways:
 1. Through running the `build` process of the splunk_eventgen pypi module
 2. Downloading the SPL direct from [splunkbase](https://splunkbase.splunk.com/app/1924/)
 
-###### Gerating the SPL file
+#### Gerating the SPL file
 In order to generate the SPL file, install Eventgen through PyPI with the instruction above.
 Once you have Eventgen installed, run:
 
@@ -130,14 +158,17 @@ Once you have Eventgen installed, run:
 $ splunk_eventgen build --destination <DESIRED_PATH_TO_OUTPUT_SPL_FILE>
 ```
 
-###### Finishing the Install
+#### Finishing the Install
+
+>:warning: For upgrading SA-eventgen from 6.x to 7.x, please refer to [this guide](UPGRADE.md#sa-app-upgrade-to-7).
+
 With the generated / downloaded SPL file, follow these steps to install:
 1. Log in to Splunk Web and navigate to Apps > Manage Apps.
 2. Click "Install app from file".
 3. Navigate to the path where your local SPL file is and select.
 4. Restart Splunk after you have been notified of a successful installation.
 
-Before you start Eventgen, confirm that you have successfully installed SA-Eventgen: 
+Before you start Eventgen, confirm that you have successfully installed SA-Eventgen:
 
 You should see SA-Eventgen App in SplunkWeb.
 ![Local Image](./images/splunk_web_sa_eventgen.png)
@@ -150,7 +181,7 @@ Additionally, you'll see SA-Eventgen in your Splunk apps installation directory:
 $ cd ${SPLUNK_HOME}/etc/apps
 ```
 
-### First Run
+#### First Run
 If SA-Eventgen App is correctly installed, there is no additional configuration required. SA-Eventgen app will automatically identify with any apps with eventgen.conf.
 
 To start generating data, simply enable the SA-Eventgen modinput by going to Settings > Data Inputs > SA-Eventgen and by clicking "enable" on the default modular input stanza.