From d51884b98aeff85e1fcc09fe79c67a5c2b586b1e Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 9 May 2016 17:20:23 +0200 Subject: [PATCH] Move "beat" (ex-"shipper") config settings to be top level (#1570) * Move "beat" (ex-shipper) config settings to be top level Part of #1417. Done in a non-BWC way, because I think it's not worth complicating the code. I'd rather have a script that migrates the configuration. * Make winlogbeat accept the new top-level settings The list has grown quite long, switching to ucfg validation for this would be nice. Also added explicit key names to config. --- CHANGELOG.asciidoc | 3 + filebeat/filebeat.yml | 101 +++++++++--------- filebeat/tests/system/config/filebeat.yml.j2 | 44 ++++---- libbeat/beat/beat.go | 10 +- libbeat/cfgfile/cfgfile_test.go | 2 +- libbeat/docs/shipperconfig.asciidoc | 89 ++++++++------- libbeat/etc/libbeat.yml | 101 +++++++++--------- libbeat/outputs/elasticsearch/output_test.go | 8 +- libbeat/publisher/publish.go | 10 +- libbeat/tests/system/config/mockbeat.yml.j2 | 47 ++++---- metricbeat/metricbeat.yml | 101 +++++++++--------- packetbeat/packetbeat.yml | 101 +++++++++--------- .../tests/system/config/packetbeat.yml.j2 | 57 +++++----- .../system/config/topbeat-input-invalid.yml | 29 +++-- topbeat/tests/system/config/topbeat-old.yml | 1 - topbeat/tests/system/config/topbeat.yml.j2 | 29 +++-- topbeat/topbeat.template-es2x.json | 3 + topbeat/topbeat.yml | 101 +++++++++--------- winlogbeat/config/config.go | 7 +- winlogbeat/config/config_test.go | 5 +- .../tests/system/config/winlogbeat.yml.j2 | 33 +++--- winlogbeat/winlogbeat.yml | 101 +++++++++--------- 22 files changed, 488 insertions(+), 495 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1e52bc4f982f..3a19dcaa3cc2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,9 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- All configuration settings under `shipper:` are moved to be top level configuration settings. I.e. + `shipper.name:` becomes `name:` in the configuration file. #1570 + *Packetbeat* *Topbeat* diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 1d82d8e8e800..24f7e9637644 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -373,57 +373,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging ######################################### diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 48f55a067d1b..3e29627d0415 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -69,30 +69,28 @@ filebeat: ############################# Shipper ############################################ -shipper: - - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - name: {{shipperName}} - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - tags: [ - {%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - {%- endif -%}] - - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - # ignore_outgoing: true +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: {{shipperName}} + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ +{%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} +{%- endif -%}] + + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +# ignore_outgoing: true ############################# Output ############################################ diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 8a94749e087e..7f9ad254a787 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -111,11 +111,11 @@ type Beat struct { // BeatConfig struct contains the basic configuration of every beat type BeatConfig struct { - Output map[string]*common.Config - Logging logp.Logging - Shipper publisher.ShipperConfig - Filters []filter.FilterConfig - Path paths.Path + Shipper publisher.ShipperConfig `config:",inline"` + Output map[string]*common.Config `config:"output"` + Logging logp.Logging `config:"logging"` + Filters []filter.FilterConfig `config:"filters"` + Path paths.Path `config:"path"` } // Run initializes and runs a Beater implementation. name is the name of the diff --git a/libbeat/cfgfile/cfgfile_test.go b/libbeat/cfgfile/cfgfile_test.go index 29b5e143cb3d..c010f9725129 100644 --- a/libbeat/cfgfile/cfgfile_test.go +++ b/libbeat/cfgfile/cfgfile_test.go @@ -83,7 +83,7 @@ func TestExpandEnv(t *testing.T) { {`password: "abc$${!"`, `password: "abc${!"`, ""}, // The full error contains the line number. - {"shipper:\n name: ${var", "", "failure while expanding environment " + + {"test:\n name: ${var", "", "failure while expanding environment " + "variables in config.yml at line=2, unterminated brace"}, } diff --git a/libbeat/docs/shipperconfig.asciidoc b/libbeat/docs/shipperconfig.asciidoc index 1e902719a4b0..a600eb7501df 100644 --- a/libbeat/docs/shipperconfig.asciidoc +++ b/libbeat/docs/shipperconfig.asciidoc @@ -11,57 +11,56 @@ ////////////////////////////////////////////////////////////////////////// [[configuration-shipper]] -=== Shipper +=== Top level configuration options -The `shipper` section contains configuration options for the Beat and some +The top level section contains configuration options for the Beat and some general settings that control its behaviour. Here is an example configuration: [source,yaml] ------------------------------------------------------------------------------ -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - tags: ["service-X", "web-tier"] - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - topology_expire: 15 - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: ["service-X", "web-tier"] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +topology_expire: 15 + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: +#paths: +# - "/usr/share/GeoIP/GeoLiteCity.dat" +# - "/usr/local/var/GeoIP/GeoLiteCity.dat" ------------------------------------------------------------------------------ -==== Shipper Options +==== Top Level Options -You can specify the following options under the `shipper` section: +You can specify the following options: ===== name The name of the Beat. If this option is empty, the `hostname` of the server is -used. The name is included as the `shipper` field in each published transaction. You can +used. The name is included as the `beat.name` field in each published transaction. You can use the name to group all transactions sent by a single Beat. At startup, each Beat can publish its IP, port, and name to Elasticsearch. This information @@ -81,8 +80,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------ -shipper: - name: "my-shipper" +name: "my-shipper" ------------------------------------------------------------------------------ ===== tags @@ -97,8 +95,7 @@ Example: [source,yaml] -------------------------------------------------------------------------------- -shipper: - tags: ["my-service", "hardware", "test"] +tags: ["my-service", "hardware", "test"] -------------------------------------------------------------------------------- [[libbeat-configuration-fields]] @@ -114,8 +111,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------ -shipper: - fields: {project: "myproject", instance-id: "574734885120952459"} +fields: {project: "myproject", instance-id: "574734885120952459"} ------------------------------------------------------------------------------ ===== fields_under_root @@ -129,11 +125,10 @@ Example: [source,yaml] ------------------------------------------------------------------------------ -shipper: - fields_under_root: true - fields: - instance_id: i-10a64379 - region: us-east-1 +fields_under_root: true +fields: + instance_id: i-10a64379 + region: us-east-1 ------------------------------------------------------------------------------ ===== ignore_outgoing diff --git a/libbeat/etc/libbeat.yml b/libbeat/etc/libbeat.yml index 9354264474e2..37b74d41aff7 100644 --- a/libbeat/etc/libbeat.yml +++ b/libbeat/etc/libbeat.yml @@ -180,57 +180,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging ######################################### diff --git a/libbeat/outputs/elasticsearch/output_test.go b/libbeat/outputs/elasticsearch/output_test.go index 4070e3f93b19..a13a0f301603 100644 --- a/libbeat/outputs/elasticsearch/output_test.go +++ b/libbeat/outputs/elasticsearch/output_test.go @@ -102,7 +102,7 @@ func TestOneEvent(t *testing.T) { event["dst_port"] = 6379 event["src_ip"] = "192.168.22.2" event["src_port"] = 6378 - event["shipper"] = "appserver1" + event["name"] = "appserver1" r := common.MapStr{} r["request"] = "MGET key1" r["response"] = "value1" @@ -140,7 +140,7 @@ func TestOneEvent(t *testing.T) { }() params := map[string]string{ - "q": "shipper:appserver1", + "q": "name:appserver1", } _, resp, err := client.SearchURI(index, "", params) @@ -174,7 +174,7 @@ func TestEvents(t *testing.T) { event["dst_port"] = 6379 event["src_ip"] = "192.168.22.2" event["src_port"] = 6378 - event["shipper"] = "appserver1" + event["name"] = "appserver1" r := common.MapStr{} r["request"] = "MGET key1" r["response"] = "value1" @@ -210,7 +210,7 @@ func TestEvents(t *testing.T) { output.randomClient().Refresh(index) params := map[string]string{ - "q": "shipper:appserver1", + "q": "name:appserver1", } defer func() { diff --git a/libbeat/publisher/publish.go b/libbeat/publisher/publish.go index 42d5645db729..8f02d8f8b14a 100644 --- a/libbeat/publisher/publish.go +++ b/libbeat/publisher/publish.go @@ -81,11 +81,11 @@ type Publisher struct { type ShipperConfig struct { common.EventMetadata `config:",inline"` // Fields and tags to add to each event. - Name string - RefreshTopologyFreq time.Duration `config:"refresh_topology_freq"` - Ignore_outgoing bool `config:"ignore_outgoing"` - Topology_expire int `config:"topology_expire"` - Geoip common.Geoip `config:"geoip"` + Name string `config:"name"` + RefreshTopologyFreq time.Duration `config:"refresh_topology_freq"` + Ignore_outgoing bool `config:"ignore_outgoing"` + Topology_expire int `config:"topology_expire"` + Geoip common.Geoip `config:"geoip"` // internal publisher queue sizes QueueSize *int `config:"queue_size"` diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 38fdad0fee0e..8d10201285c2 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -4,30 +4,29 @@ mockbeat: -############################# Shipper ############################################ -shipper: - - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - tags: [ - {%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - {%- endif -%}] - - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - # ignore_outgoing: true +############################# General ############################################ + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ +{%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} +{%- endif -%}] + + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +# ignore_outgoing: true diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index fc47dd9d2604..cfcf55c5e202 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -264,57 +264,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging ######################################### diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 0af0ed4cb3ac..81d7bd6ab9db 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -357,57 +357,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging ######################################### diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 9f843774230f..b2e012b23293 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -164,37 +164,36 @@ filters: # vim: set ft=jinja: ############################# Shipper ######################################### -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - tags: [ - {%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - {%- endif -%} - ] - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - # ignore_outgoing: true +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +# ignore_outgoing: true {% if geoip_paths is not none %} - geoip: - paths: [ - {%- for path in geoip_paths -%} - "{{ beat.working_dir + '/' + path }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] {%- endif %} ############################# Output ########################################## diff --git a/topbeat/tests/system/config/topbeat-input-invalid.yml b/topbeat/tests/system/config/topbeat-input-invalid.yml index ee547a5480fe..3fce22437590 100644 --- a/topbeat/tests/system/config/topbeat-input-invalid.yml +++ b/topbeat/tests/system/config/topbeat-input-invalid.yml @@ -41,21 +41,20 @@ output: ############################# Shipper ######################################### -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true ############################# Logging ######################################### diff --git a/topbeat/tests/system/config/topbeat-old.yml b/topbeat/tests/system/config/topbeat-old.yml index 56c521397c2a..f09ef9b3df3c 100644 --- a/topbeat/tests/system/config/topbeat-old.yml +++ b/topbeat/tests/system/config/topbeat-old.yml @@ -38,4 +38,3 @@ output: ############################# Shipper ######################################### -shipper: diff --git a/topbeat/tests/system/config/topbeat.yml.j2 b/topbeat/tests/system/config/topbeat.yml.j2 index 554a870f5c4f..f9132d5ceca5 100644 --- a/topbeat/tests/system/config/topbeat.yml.j2 +++ b/topbeat/tests/system/config/topbeat.yml.j2 @@ -51,21 +51,20 @@ output: ############################# Shipper ######################################### -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true ############################# Logging ######################################### diff --git a/topbeat/topbeat.template-es2x.json b/topbeat/topbeat.template-es2x.json index 70c823429ce4..8148d859bcc2 100644 --- a/topbeat/topbeat.template-es2x.json +++ b/topbeat/topbeat.template-es2x.json @@ -119,6 +119,9 @@ "files": { "type": "long" }, + "free": { + "type": "long" + }, "free_files": { "type": "long" }, diff --git a/topbeat/topbeat.yml b/topbeat/topbeat.yml index 829d2820bdf6..ff13c82b1a23 100644 --- a/topbeat/topbeat.yml +++ b/topbeat/topbeat.yml @@ -206,57 +206,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging ######################################### diff --git a/winlogbeat/config/config.go b/winlogbeat/config/config.go index 12f6cc055dc7..a93b511d18d3 100644 --- a/winlogbeat/config/config.go +++ b/winlogbeat/config/config.go @@ -34,7 +34,12 @@ type Settings struct { // Validate validates the Settings data and returns an error describing // all problems or nil if there are none. func (s Settings) Validate() error { - validKeys := []string{"filters", "logging", "output", "shipper", "winlogbeat"} + validKeys := []string{ + "fields", "fields_under_root", "tags", + "name", "refresh_topology_freq", "ignore_outgoing", "topology_expire", "geoip", + "queue_size", "bulk_queue_size", "max_procs", + "filters", "logging", "output", "path", "winlogbeat", + } sort.Strings(validKeys) // Check for invalid top-level keys. diff --git a/winlogbeat/config/config_test.go b/winlogbeat/config/config_test.go index f26d82f23ba4..e28406656e09 100644 --- a/winlogbeat/config/config_test.go +++ b/winlogbeat/config/config_test.go @@ -44,8 +44,9 @@ func TestConfigValidate(t *testing.T) { }, map[string]interface{}{"other": "value"}, }, - "1 error: Invalid top-level key 'other' found. Valid keys are " + - "filters, logging, output, shipper, winlogbeat", + "1 error: Invalid top-level key 'other' found. Valid keys are bulk_queue_size, " + + "fields, fields_under_root, filters, geoip, ignore_outgoing, logging, max_procs, " + + "name, output, path, queue_size, refresh_topology_freq, tags, topology_expire, winlogbeat", }, { WinlogbeatConfig{}, diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2 index db12a394711b..fe308291d73d 100644 --- a/winlogbeat/tests/system/config/winlogbeat.yml.j2 +++ b/winlogbeat/tests/system/config/winlogbeat.yml.j2 @@ -64,24 +64,23 @@ output: ############################# Shipper ######################################### -shipper: - {%- if shipper_name %} - name: {{ shipper_name }} - {% endif %} +{%- if shipper_name %} +name: {{ shipper_name }} +{% endif %} - {%- if tags %} - tags: - {% for tag in tags -%} - - {{ tag }} - {% endfor -%} - {% endif %} +{%- if tags %} +tags: + {% for tag in tags -%} + - {{ tag }} + {% endfor -%} +{% endif %} - {%- if fields %} - {% if fields_under_root %}fields_under_root: true{% endif %} - fields: - {% for k, v in fields.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {% endif %} +{%- if fields %} +{% if fields_under_root %}fields_under_root: true{% endif %} +fields: + {% for k, v in fields.items() -%} + {{ k }}: {{ v }} + {% endfor -%} +{% endif %} # vim: set ft=jinja: diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 23f69915c338..f64522d9c271 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -208,57 +208,56 @@ output: #pretty: false -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # If this option is set to true, the custom fields are stored as top-level - # fields in the output document instead of being grouped under a fields - # sub-dictionary. Default is false. - #fields_under_root: false - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true - - # How often (in seconds) shippers are publishing their IPs to the topology map. - # The default is 10 seconds. - #refresh_topology_freq: 10 - - # Expiration time (in seconds) of the IPs published by a shipper to the topology map. - # All the IPs will be deleted afterwards. Note, that the value must be higher than - # refresh_topology_freq. The default is 15 seconds. - #topology_expire: 15 - - # Internal queue size for single events in processing pipeline - #queue_size: 1000 - - # Sets the maximum number of CPUs that can be executing simultaneously. The - # default is the number of logical CPUs available in the system. - #max_procs: - - # Configure local GeoIP database support. - # If no paths are not configured geoip is disabled. - #geoip: - #paths: - # - "/usr/share/GeoIP/GeoLiteCity.dat" - # - "/usr/local/var/GeoIP/GeoLiteCity.dat" +############################# General ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# Configure local GeoIP database support. +# If no paths are not configured geoip is disabled. +#geoip: + #paths: + # - "/usr/share/GeoIP/GeoLiteCity.dat" + # - "/usr/local/var/GeoIP/GeoLiteCity.dat" ############################# Logging #########################################