Added short versions for the configuration files

For now, the short files are named like `beatname.short.yml` and I kept the
`beatname.yml` to be the full one.  The intention is to switch them around. The
short ones to become the default and the long ones to be something like
`beatname.full.yml` or similar.

But before doing that I wanted the others in the team to have a look. The
decision of what to include in the short version is fairly arbitrary, I just
did it based on what I thought makes most sense.

I kept the ES and LS outputs in the configuration file. I considered having
only ES for all the beats except Filebeat, but that would complicate the
generation part and I think it's nice to highlight that we support multiple
outputs.

filebeat/etc/beat.short.yml

@@ -0,0 +1,58 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- input_type: log
+
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /var/log/*.log
+    #- c:\programdata\elasticsearch\logs\*
+
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  #exclude_lines: ["^DBG"]
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  #include_lines: ["^ERR", "^WARN"]
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #exclude_files: [".gz$"]
+
+  # Optional additional fields. These field can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+  ### Multiline options
+
+  # Mutiline can be used for log messages spanning multiple lines. This is common
+  # for Java Stack Traces or C-Line Continuation
+
+  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+  #multiline.pattern: ^\[
+
+  # Defines if the pattern set under pattern should be negated or not. Default is false.
+  #multiline.negate: false
+
+  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+  #multiline.match: after
+

filebeat/etc/beat.yml

@@ -1,10 +1,19 @@
-###################### Filebeat Configuration Example #########################
+##################$$$###### Filebeat Configuration ############################
+
+# This file is a full configuration example documenting all non-deprecated
+# options in comments. For a shorter configuration example, that contains only
+# the most common options, please see filebeat.short.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
 
 #=========================== Filebeat prospectors =============================
 
 # List of prospectors to fetch data.
 filebeat.prospectors:
-# Each - is a prospector. Below are the prospector specific configurations
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
 
 # Type of the files. Based on this the way the file is read is decided.
 # The different types cannot be mixed in one prospector
@@ -32,25 +41,6 @@ filebeat.prospectors:
   #    hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ...
   #encoding: plain
 
-  # Decode JSON options. Enable this if your logs are structured in JSON.
-  # JSON key on which to apply the line filtering and multiline settings. This key
-  # must be top level and its value must be string, otherwise it is ignored. If
-  # no text key is defined, the line filtering and multiline features cannot be used.
-  #json.message_key:
-
-  # By default, the decoded JSON is placed under a "json" key in the output document.
-  # If you enable this setting, the keys are copied top level in the output document.
-  #json.keys_under_root: false
-
-  # If keys_under_root and this setting are enabled, then the values from the decoded
-  # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
-  # in case of conflicts.
-  #json.overwrite_keys: false
-
-  # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
-  # unmarshaling errors or when a text key is defined in the configuration but cannot
-  # be used.
-  #json.add_error_key: false
 
   # Exclude lines. A list of regular expressions to match. It drops the lines that are
   # matching any regular expression from the list. The include_lines is called before
@@ -106,9 +96,33 @@ filebeat.prospectors:
   # This is especially useful for multiline log messages which can get large.
   #max_bytes: 10485760
 
+  ### JSON configuration
+
+  # Decode JSON options. Enable this if your logs are structured in JSON.
+  # JSON key on which to apply the line filtering and multiline settings. This key
+  # must be top level and its value must be string, otherwise it is ignored. If
+  # no text key is defined, the line filtering and multiline features cannot be used.
+  #json.message_key:
+
+  # By default, the decoded JSON is placed under a "json" key in the output document.
+  # If you enable this setting, the keys are copied top level in the output document.
+  #json.keys_under_root: false
+
+  # If keys_under_root and this setting are enabled, then the values from the decoded
+  # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
+  # in case of conflicts.
+  #json.overwrite_keys: false
+
+  # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
+  # unmarshaling errors or when a text key is defined in the configuration but cannot
+  # be used.
+  #json.add_error_key: false
+
+  ### Multiline options
+
   # Mutiline can be used for log messages spanning multiple lines. This is common
   # for Java Stack Traces or C-Line Continuation
-  
+
   # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
   #multiline.pattern: ^\[
 

filebeat/filebeat.short.yml

@@ -0,0 +1,113 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- input_type: log
+
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /var/log/*.log
+    #- c:\programdata\elasticsearch\logs\*
+
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  #exclude_lines: ["^DBG"]
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  #include_lines: ["^ERR", "^WARN"]
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #exclude_files: [".gz$"]
+
+  # Optional additional fields. These field can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+  ### Multiline options
+
+  # Mutiline can be used for log messages spanning multiple lines. This is common
+  # for Java Stack Traces or C-Line Continuation
+
+  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+  #multiline.pattern: ^\[
+
+  # Defines if the pattern set under pattern should be negated or not. Default is false.
+  #multiline.negate: false
+
+  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+  #multiline.match: after
+
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+#================================ Outputs =====================================
+
+# Configure what outputs to use when sending the data collected by the beat.
+# Multiple outputs may be used.
+
+#-------------------------- Elasticsearch output ------------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["localhost:9200"]
+
+  # Template name. By default the template name is filebeat.
+  template.name: "filebeat"
+
+  # Path to template file
+  template.path: "filebeat.template.json"
+
+  # Overwrite existing template
+  template.overwrite: false
+
+#----------------------------- Logstash output --------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+  # Optional TLS. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #tls.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for TLS client authentication
+  #tls.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #tls.certificate_key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is error.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: error

filebeat/filebeat.yml

@@ -1,10 +1,19 @@
-###################### Filebeat Configuration Example #########################
+##################$$$###### Filebeat Configuration ############################
+
+# This file is a full configuration example documenting all non-deprecated
+# options in comments. For a shorter configuration example, that contains only
+# the most common options, please see filebeat.short.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
 
 #=========================== Filebeat prospectors =============================
 
 # List of prospectors to fetch data.
 filebeat.prospectors:
-# Each - is a prospector. Below are the prospector specific configurations
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
 
 # Type of the files. Based on this the way the file is read is decided.
 # The different types cannot be mixed in one prospector
@@ -32,25 +41,6 @@ filebeat.prospectors:
   #    hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ...
   #encoding: plain
 
-  # Decode JSON options. Enable this if your logs are structured in JSON.
-  # JSON key on which to apply the line filtering and multiline settings. This key
-  # must be top level and its value must be string, otherwise it is ignored. If
-  # no text key is defined, the line filtering and multiline features cannot be used.
-  #json.message_key:
-
-  # By default, the decoded JSON is placed under a "json" key in the output document.
-  # If you enable this setting, the keys are copied top level in the output document.
-  #json.keys_under_root: false
-
-  # If keys_under_root and this setting are enabled, then the values from the decoded
-  # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
-  # in case of conflicts.
-  #json.overwrite_keys: false
-
-  # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
-  # unmarshaling errors or when a text key is defined in the configuration but cannot
-  # be used.
-  #json.add_error_key: false
 
   # Exclude lines. A list of regular expressions to match. It drops the lines that are
   # matching any regular expression from the list. The include_lines is called before
@@ -106,9 +96,33 @@ filebeat.prospectors:
   # This is especially useful for multiline log messages which can get large.
   #max_bytes: 10485760
 
+  ### JSON configuration
+
+  # Decode JSON options. Enable this if your logs are structured in JSON.
+  # JSON key on which to apply the line filtering and multiline settings. This key
+  # must be top level and its value must be string, otherwise it is ignored. If
+  # no text key is defined, the line filtering and multiline features cannot be used.
+  #json.message_key:
+
+  # By default, the decoded JSON is placed under a "json" key in the output document.
+  # If you enable this setting, the keys are copied top level in the output document.
+  #json.keys_under_root: false
+
+  # If keys_under_root and this setting are enabled, then the values from the decoded
+  # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
+  # in case of conflicts.
+  #json.overwrite_keys: false
+
+  # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
+  # unmarshaling errors or when a text key is defined in the configuration but cannot
+  # be used.
+  #json.add_error_key: false
+
+  ### Multiline options
+
   # Mutiline can be used for log messages spanning multiple lines. This is common
   # for Java Stack Traces or C-Line Continuation
-  
+
   # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
   #multiline.pattern: ^\[
 
@@ -336,7 +350,6 @@ output.elasticsearch:
 
 
 #----------------------------- Logstash output --------------------------------
-### Logstash as output
 #output.logstash:
   # The Logstash hosts
   #hosts: ["localhost:5044"]
@@ -411,15 +424,14 @@ output.elasticsearch:
 # Under Windows systems, the log files are per default sent to the file output,
 # under all other system per default to syslog.
 
+# Sets log level. The default log level is error.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: error
 
-  # Enable debug output for selected components. To enable all selectors use ["*"]
-  # Other available selectors are beat, publish, service
-  # Multiple selectors can be chained.
-  #selectors: [ ]
-
-  # Sets log level. The default log level is error.
-  # Available log levels are: critical, error, warning, info, debug
-  #level: error
+# Enable debug output for selected components. To enable all selectors use ["*"]
+# Other available selectors are beat, publish, service
+# Multiple selectors can be chained.
+#logging.selectors: [ ]
 
 # Send all logging output to syslog. The default is false.
 #logging.to_syslog: true