splunk · li-wu · Apr 6, 2020 · Mar 20, 2020 · Mar 23, 2020 · Mar 23, 2020
diff --git a/Makefile b/Makefile
@@ -51,12 +51,15 @@ test_helper:
 	@echo 'Installing docker-compose'
 	bash install_docker_compose.sh
 
+	@echo 'Build a docker image'
+	docker build -t provision_splunk:latest -f tests/large/provision/Dockerfile tests/large/provision
+
 	@echo 'Start container with splunk'
 	docker-compose -f tests/large/provision/docker-compose.yml up &
 
 	sleep 120
 	@echo 'Provision splunk container'
-	docker-compose -f tests/large/provision/docker-compose.yml exec -T splunk sh -c 'cd /opt/splunk;./provision.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart'
+	docker exec --user splunk provision_splunk_1 sh -c 'cd /opt/splunk;./provision.sh;./add_httpevent_collector.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart'
 
 run_tests:
 	@echo 'Running the super awesome tests'

diff --git a/splunk_eventgen/lib/eventgenconfig.py b/splunk_eventgen/lib/eventgenconfig.py
@@ -1,3 +1,4 @@
+import copy
 import datetime
 import json
 import logging.handlers
@@ -520,15 +521,22 @@ def parse(self):
             if os.path.exists(s.sampleDir):
                 sampleFiles = os.listdir(s.sampleDir)
                 for sample in sampleFiles:
-                    results = re.match(s.name, sample)
+                    sample_name = s.name
+                    # If we expect a .csv, append it to the file name - regex matching must include the extension
+                    if s.sampletype == "csv" and not s.name.endswith(".csv"):
+                        sample_name = s.name + ".csv"
+                    results = re.match(sample_name, sample)
                     if results:
-                        logger.debug("Matched file {0} with sample name {1}".format(results.group(0), s.name))
-                        samplePath = os.path.join(s.sampleDir, sample)
-                        if os.path.isfile(samplePath):
-                            logger.debug(
-                                "Found sample file '%s' for app '%s' using config '%s' with priority '%s'" %
-                                (sample, s.app, s.name, s._priority) + "; adding to list")
-                            foundFiles.append(samplePath)
+                        # Make sure the stanza name/regex matches the entire file name
+                        match_start, match_end = results.regs[0]
+                        if match_end - match_start == len(sample):
+                            logger.debug("Matched file {0} with sample name {1}".format(results.group(0), s.name))
+                            samplePath = os.path.join(s.sampleDir, sample)
+                            if os.path.isfile(samplePath):
+                                logger.debug(
+                                    "Found sample file '%s' for app '%s' using config '%s' with priority '%s'" %
+                                    (sample, s.app, s.name, s._priority) + "; adding to list")
+                                foundFiles.append(samplePath)
 
             # If we didn't find any files, log about it
             if len(foundFiles) == 0:
@@ -539,8 +547,8 @@ def parse(self):
                     tempsamples2.append(s)
 
             for f in foundFiles:
-                if s.name in f:
-                    news = s
+                if re.search(s.name, f):
+                    news = copy.copy(s)
                     news.filePath = f
                     # 12/3/13 CS TODO These are hard coded but should be handled via the modular config system
                     # Maybe a generic callback for all plugins which will modify sample based on the filename

diff --git a/tests/large/conf/eventgen_sample_regex_csv.conf b/tests/large/conf/eventgen_sample_regex_csv.conf
@@ -0,0 +1,14 @@
+[timeorder.*]
+sampleDir = ../sample
+mode = sample
+sampletype = csv
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]
diff --git a/tests/large/conf/eventgen_sample_regex_integer.conf b/tests/large/conf/eventgen_sample_regex_integer.conf
@@ -0,0 +1,15 @@
+[sample\d]
+sampleDir = ../sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]
diff --git a/tests/large/conf/eventgen_sample_regex_wildcard.conf b/tests/large/conf/eventgen_sample_regex_wildcard.conf
@@ -0,0 +1,15 @@
+[sample.*]
+sampleDir = ../sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]
diff --git a/tests/large/provision/Dockerfile b/tests/large/provision/Dockerfile
@@ -1,11 +1,8 @@
-FROM splunk/splunk:7.0.3-monitor
+FROM splunk/splunk:7.3-debian
 
-# https://superuser.com/questions/1423486/issue-with-fetching-http-deb-debian-org-debian-dists-jessie-updates-inrelease
-RUN printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list
-
-RUN apt-get update
+RUN sudo apt-get update
 
 RUN echo "installing docker dependencies and development tools" && \
-    apt-get --assume-yes install curl vim
+    sudo apt-get --assume-yes install curl vim
 
-COPY provision.sh /opt/splunk/
+COPY ["provision.sh", "add_httpevent_collector.sh", "/opt/splunk/"]
diff --git a/tests/large/provision/add_httpevent_collector.sh b/tests/large/provision/add_httpevent_collector.sh
@@ -0,0 +1,5 @@
+HTTP_INPUTS_PATH=/opt/splunk/etc/apps/search/local/inputs.conf
+echo "[http://test]" >> $HTTP_INPUTS_PATH
+echo "disabled = 0" >> $HTTP_INPUTS_PATH
+echo "token = 00000000-0000-0000-0000-000000000000" >> $HTTP_INPUTS_PATH
+echo "indexes = main,test_0,test_1" >> $HTTP_INPUTS_PATH
diff --git a/tests/large/provision/docker-compose.yml b/tests/large/provision/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.3"
 services:
   splunk:
     hostname: eventgensplunk
-    build: .
+    image: provision_splunk:latest
     ports:
       - 8000:8000
       - 8089:8089
@@ -12,8 +12,10 @@ services:
       SPLUNK_START_ARGS: --answer-yes --no-prompt --accept-license
       # add `SHELL` env variable to make the `dircolors` happy
       SHELL: /bin/bash
+      SPLUNK_PASSWORD: changeme
     volumes:
       # the `docker` command in guest can talk to host docker daemon
       - "/var/run/docker.sock:/var/run/docker.sock"
     # to make terminal colorful
     tty: true
+
diff --git a/tests/large/sample/sample1 b/tests/large/sample/sample1
@@ -0,0 +1,12 @@
+2014-01-04 20:00:00 WINDBAG Event 1 of 12 randint @@integer
+2014-01-04 20:00:01 WINDBAG Event 2 of 12 randint @@integer
+2014-01-04 20:00:02 WINDBAG Event 3 of 12 randint @@integer
+2014-01-04 20:00:03 WINDBAG Event 4 of 12 randint @@integer
+2014-01-04 20:00:03 WINDBAG Event 5 of 12 randint @@integer
+2014-01-04 20:00:04 WINDBAG Event 6 of 12 randint @@integer
+2014-01-04 20:00:05 WINDBAG Event 7 of 12 randint @@integer
+2014-01-04 20:00:06 WINDBAG Event 8 of 12 randint @@integer
+2014-01-04 20:00:08 WINDBAG Event 9 of 12 randint @@integer
+2014-01-04 20:00:20 WINDBAG Event 10 of 12 randint @@integer
+2014-01-04 20:00:21 WINDBAG Event 11 of 12 randint @@integer
+2014-01-04 20:00:21 WINDBAG Event 12 of 12 randint @@integer
diff --git a/tests/large/sample/sample2 b/tests/large/sample/sample2
@@ -0,0 +1,12 @@
+2014-01-04 20:00:00 WINDBAG Event 1 of 12 randint @@integer
+2014-01-04 20:00:01 WINDBAG Event 2 of 12 randint @@integer
+2014-01-04 20:00:02 WINDBAG Event 3 of 12 randint @@integer
+2014-01-04 20:00:03 WINDBAG Event 4 of 12 randint @@integer
+2014-01-04 20:00:03 WINDBAG Event 5 of 12 randint @@integer
+2014-01-04 20:00:04 WINDBAG Event 6 of 12 randint @@integer
+2014-01-04 20:00:05 WINDBAG Event 7 of 12 randint @@integer
+2014-01-04 20:00:06 WINDBAG Event 8 of 12 randint @@integer
+2014-01-04 20:00:08 WINDBAG Event 9 of 12 randint @@integer
+2014-01-04 20:00:20 WINDBAG Event 10 of 12 randint @@integer
+2014-01-04 20:00:21 WINDBAG Event 11 of 12 randint @@integer
+2014-01-04 20:00:21 WINDBAG Event 12 of 12 randint @@integer
diff --git a/tests/large/sample/timeorder_regex.csv b/tests/large/sample/timeorder_regex.csv
@@ -0,0 +1,11 @@
+_time,_raw,index,host,source,sourcetype
+2015-08-18T16:28:54.695-0700,"127.0.0.1 - admin [18/Aug/2015:16:28:54.695 -0700] ""GET /en-US/api/shelper?snippet=true&snippetEmbedJS=false&namespace=search&search=search+index%3D_internal+%7C+fields+_time%2C+_raw%2C+index%2C+host%2C+source%2C+sourcetype+&useTypeahead=true&useAssistant=true&showCommandHelp=true&showCommandHistory=true&showFieldInfo=false&_=1439940537886 HTTP/1.1"" 200 994 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 55d3bfb6b17f7ff8270d50 33ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access
+2015-08-18T16:28:54.569-0700,"2015-08-18 16:28:54,569 INFO	streams_utils:24 - utils::readAsJson:: /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/apps",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log
+2015-08-18T16:28:54.568-0700,"2015-08-18 16:28:54,568 INFO	streams_utils:74 - create dir /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log
+2015-08-18T16:28:54.564-0700,"127.0.0.1 - - [18/Aug/2015:16:28:54.564 -0700] ""GET /en-us/custom/splunk_app_stream/ping/ HTTP/1.1"" 200 311 """" """" - 55d3bfb6907f7ff805f710 5ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access
+2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224/summary?output_mode=json&min_freq=0&_=1439940537880 HTTP/1.1"" 200 503 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 6ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access
+2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/services/search/jobs/1439940529.1846224/timeline?offset=0&count=1000&_=1439940537881 HTTP/1.1"" 200 349 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access
+2015-08-18T16:28:52.754-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.754 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224?output_mode=json&_=1439940537879 HTTP/1.1"" 200 1543 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access
+2015-08-18T16:28:52.270-0700,"2015-08-18 16:28:52,270 ERROR pid=16324 tid=MainThread file=__init__.py:execute:957 | Execution failed: [HTTP 401] Client is not authenticated
+2015-08-18T16:28:52.268-0700,"127.0.0.1 - - [18/Aug/2015:16:28:52.268 -0700] ""GET /services/shcluster/config/config HTTP/1.0"" 401 148 - - - 0ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_access.log,splunkd_access
+2015-08-18T16:28:52.247-0700,"2015-08-18 16:28:52,247 INFO pid=16324 tid=MainThread file=__init__.py:execute:906 | Execute called",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/python_modular_input.log,python_modular_input
diff --git a/tests/large/sample/tutorial1 → tests/large/sample/tutorial1.csv b/tests/large/sample/tutorial1 → tests/large/sample/tutorial1.csv
diff --git a/tests/large/test_mode_replay.py b/tests/large/test_mode_replay.py
@@ -52,7 +52,7 @@ def test_mode_replay_backfill_greater_interval(eventgen_test_helper):
 
 
 def test_mode_replay_tutorial1(eventgen_test_helper):
-    """Test the replay mode with csv for sample file sample.tutorial1. https://github.com/splunk/eventgen/issues/244"""
+    """Test the replay mode with csv for sample file sample.tutorial1.csv"""
     events = eventgen_test_helper('eventgen_tutorial1.conf').get_events()
     assert len(events) == 2019
 

diff --git a/tests/large/test_mode_sample.py b/tests/large/test_mode_sample.py
@@ -105,3 +105,21 @@ def test_mode_sample_generator_workers(eventgen_test_helper):
     """Test sample mode with generatorWorkers = 5, end = 5 and count = 10"""
     events = eventgen_test_helper("eventgen_sample_generatorWorkers.conf").get_events()
     assert len(events) == 50
+
+
+def test_mode_sample_regex_integer(eventgen_test_helper):
+    """Test sample mode with a regex pattern in the stanza name ('sample\d')"""
+    events = eventgen_test_helper("eventgen_sample_regex_integer.conf").get_events()
+    assert len(events) == 24
+
+
+def test_mode_sample_regex_wildcard(eventgen_test_helper):
+    """tTest sample mode with a regex wildcard pattern in the stanza name ('sample*')"""
+    events = eventgen_test_helper("eventgen_sample_regex_wildcard.conf").get_events()
+    assert len(events) == 36
+
+
+def test_mode_sample_regex_csv(eventgen_test_helper):
+    """tTest sample mode with a regex wildcard pattern in the stanza name ('sample*')"""
+    events = eventgen_test_helper("eventgen_sample_regex_csv.conf").get_events()
+    assert len(events) == 20