diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2efb0b16756d..47fe0a26cd31 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -9,12 +9,53 @@ This file is generated! See etc/fields.yml and scripts/generate_field_docs.py This document describes the fields that are exported by Metricbeat. They are grouped in the following categories: +* <> * <> +* <> + +[[exported-fields-common]] +=== Common fields Fields + +Contains common fields available in all event types. + + + +==== beat.name + +The name of the Beat sending the log messages. If the shipper name is set in the configuration file, then that value is used. If it is not set, the hostname is used. + + +==== beat.hostname + +The hostname as returned by the operating system on which the Beat is running. + + +==== @timestamp + +type: date + +example: 2015-01-24 14:06:05.071000 + +format: YYYY-MM-DDTHH:MM:SS.milliZ + +required: True + +The timestamp when the log line was read. The precision is in milliseconds. The timezone is UTC. + + +==== response_time + +type: long + +required: True + +Event Reponse time in nano seconds + [[exported-fields-redis-info]] === Redis Info Stats Fields -Redis info Stats +Redis Stats @@ -33,28 +74,28 @@ Redis client stats ==== redis-info.clients.connected_clients -type: int +type: integer Number of client connections (excluding connections from slaves) ==== redis-info.clients.client_longest_output_list -type: int +type: integer Longest output list among current client connections. ==== redis-info.clients.client_biggest_input_buf -type: int +type: integer Biggest input buffer among current client connections ==== redis-info.clients.blocked_clients -type: int +type: integer Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH) @@ -106,3 +147,57 @@ type: float User CPU consumed by the background processes +[[exported-fields-mysql-status]] +=== MySQL Status Fields + +MySQL Stats + + + +[[exported-fields-mysql-status]] +=== MySQL Status Fields + +MySQL Status + + + +=== aborted Fields + +Aborted status fields + + + +==== mysql-status.aborted.Aborted_clients + +type: integer + +The number of connections that were aborted because the client died without closing the connection properly. + + +==== mysql-status.aborted.Aborted_connects + +type: integer + +The number of failed attempts to connect to the MySQL server. + + +=== bytes Fields + +Bytes stats + + + +==== mysql-status.bytes.Bytes_received + +type: integer + +The number of bytes received from all clients. + + +==== mysql-status.bytes.Bytes_sent + +type: integer + +The number of bytes sent to all clients. + + diff --git a/metricbeat/etc/fields.yml b/metricbeat/etc/fields.yml index 9b2519541f6a..d1d4977fc9db 100644 --- a/metricbeat/etc/fields.yml +++ b/metricbeat/etc/fields.yml @@ -8,11 +8,83 @@ defaults: ignore_above: 1024 sections: + - ["common", "Common fields"] - ["redis-info", "Redis Info Stats"] + - ["mysql-status", "MySQL Status"] + +common: + type: group + description: > + Contains common fields available in all event types. + fields: + - name: beat.name + description: > + The name of the Beat sending the log messages. If the shipper name is set + in the configuration file, then that value is used. If it is not set, + the hostname is used. + + - name: beat.hostname + description: > + The hostname as returned by the operating system on which the Beat is + running. + + - name: "@timestamp" + type: date + required: true + format: YYYY-MM-DDTHH:MM:SS.milliZ + example: 2015-01-24T14:06:05.071Z + description: > + The timestamp when the log line was read. The precision is in + milliseconds. The timezone is UTC. + + - name: response_time + type: long + required: true + description: > + Event Reponse time in nano seconds +mysql-status: + type: group + description: > + MySQL Stats + fields: + - name: mysql-status + type: group + description: > + MySQL Status + fields: + - name: aborted + type: group + description: > + Aborted status fields + fields: + - name: Aborted_clients + type: integer + description: > + The number of connections that were aborted because the client died without closing the connection properly. + + - name: Aborted_connects + type: integer + description: > + The number of failed attempts to connect to the MySQL server. + + - name: bytes + type: group + description: > + Bytes stats + fields: + - name: Bytes_received + type: integer + description: > + The number of bytes received from all clients. + + - name: Bytes_sent + type: integer + description: > + The number of bytes sent to all clients. redis-info: type: group description: > - Redis info Stats + Redis Stats fields: - name: redis-info type: group @@ -25,22 +97,22 @@ redis-info: Redis client stats fields: - name: connected_clients - type: int + type: integer description: > Number of client connections (excluding connections from slaves) - name: client_longest_output_list - type: int + type: integer description: > Longest output list among current client connections. - name: client_biggest_input_buf - type: int + type: integer description: > Biggest input buffer among current client connections - name: blocked_clients - type: int + type: integer description: > Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH) diff --git a/metricbeat/etc/fields_base.yml b/metricbeat/etc/fields_base.yml index 50de91c240a3..9875a3e0257b 100644 --- a/metricbeat/etc/fields_base.yml +++ b/metricbeat/etc/fields_base.yml @@ -8,4 +8,37 @@ defaults: ignore_above: 1024 sections: + - ["common", "Common fields"] - ["redis-info", "Redis Info Stats"] + - ["mysql-status", "MySQL Status"] + +common: + type: group + description: > + Contains common fields available in all event types. + fields: + - name: beat.name + description: > + The name of the Beat sending the log messages. If the shipper name is set + in the configuration file, then that value is used. If it is not set, + the hostname is used. + + - name: beat.hostname + description: > + The hostname as returned by the operating system on which the Beat is + running. + + - name: "@timestamp" + type: date + required: true + format: YYYY-MM-DDTHH:MM:SS.milliZ + example: 2015-01-24T14:06:05.071Z + description: > + The timestamp when the log line was read. The precision is in + milliseconds. The timezone is UTC. + + - name: response_time + type: long + required: true + description: > + Event Reponse time in nano seconds diff --git a/metricbeat/etc/kibana/dashboard/metricbeat-mysql.json b/metricbeat/etc/kibana/dashboard/metricbeat-mysql.json new file mode 100644 index 000000000000..3004d63859d6 --- /dev/null +++ b/metricbeat/etc/kibana/dashboard/metricbeat-mysql.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "metricbeat-mysql", + "uiStateJSON": "{}", + "panelsJSON": "[{\"id\":\"metricbeat-mysql-open\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" + } +} \ No newline at end of file diff --git a/metricbeat/etc/kibana/dashboard/redis.json b/metricbeat/etc/kibana/dashboard/metricbeat-redis.json similarity index 70% rename from metricbeat/etc/kibana/dashboard/redis.json rename to metricbeat/etc/kibana/dashboard/metricbeat-redis.json index bec8721782e4..40ebf0d3ee59 100644 --- a/metricbeat/etc/kibana/dashboard/redis.json +++ b/metricbeat/etc/kibana/dashboard/metricbeat-redis.json @@ -1,13 +1,13 @@ { - "hits": 0, - "timeRestore": false, - "description": "", - "title": "redis", - "uiStateJSON": "{\"P-2\":{\"vis\":{\"legendOpen\":false}},\"P-1\":{\"vis\":{\"legendOpen\":false}}}", - "panelsJSON": "[{\"id\":\"connected_clients\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":1},{\"id\":\"connections-received\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":3}]", - "optionsJSON": "{\"darkTheme\":false}", - "version": 1, + "hits": 0, + "timeRestore": false, + "description": "", + "title": "metricbeat-redis", + "uiStateJSON": "{\"P-2\":{\"vis\":{\"legendOpen\":false}},\"P-1\":{\"vis\":{\"legendOpen\":false}}}", + "panelsJSON": "[{\"id\":\"connected_clients\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":1},{\"id\":\"connections-received\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":3}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" } -} \ No newline at end of file +} diff --git a/metricbeat/etc/kibana/visualization/metricbeat-mysql-open.json b/metricbeat/etc/kibana/visualization/metricbeat-mysql-open.json new file mode 100644 index 000000000000..15b5344955b9 --- /dev/null +++ b/metricbeat/etc/kibana/visualization/metricbeat-mysql-open.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mysql-status.open.Open_tables\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mysql-status.open.Open_files\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mysql-status.open.Opened_tables\"}}],\"listeners\":{}}", + "description": "", + "title": "metricbeat-mysql-open", + "uiStateJSON": "{\"vis\":{\"colors\":{\"Max mysql-status.bytes.Bytes_received\":\"#F4D598\"}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"metricbeat-*\",\"query\":{\"query_string\":{\"query\":\"module:\\\"mysql\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/etc/metricbeat.template.json b/metricbeat/etc/metricbeat.template.json index 23d063ae713f..531a0f85928d 100644 --- a/metricbeat/etc/metricbeat.template.json +++ b/metricbeat/etc/metricbeat.template.json @@ -21,6 +21,9 @@ } ], "properties": { + "@timestamp": { + "type": "date" + }, "redis-info": { "properties": { "cpu": { @@ -44,6 +47,10 @@ } } } + }, + "response_time": { + "doc_values": "true", + "type": "long" } } } diff --git a/metricbeat/module/mysql/status/data.go b/metricbeat/module/mysql/status/data.go index aadf949019d2..99592e36a285 100644 --- a/metricbeat/module/mysql/status/data.go +++ b/metricbeat/module/mysql/status/data.go @@ -1,25 +1,65 @@ package status import ( - "github.com/elastic/beats/libbeat/common" - "strconv" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" ) -// Map data to MapStr +// Map data to MapStr of server stats variables: http://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html func eventMapping(status map[string]string) common.MapStr { - connections, _ := strconv.Atoi(status["Connections"]) - openTables, _ := strconv.Atoi(status["Open_tables"]) - openFiles, _ := strconv.Atoi(status["Open_files"]) - openStreams, _ := strconv.Atoi(status["Open_streams"]) - event := common.MapStr{ - "Connections": connections, - "Open_tables": openTables, - "Open_files": openFiles, - "Open_streams": openStreams, + "aborted": common.MapStr{ + + "Aborted_clients": toInt(status["Aborted_clients"]), + "Aborted_connects": toInt(status["Aborted_connects"]), + }, + "binlog": common.MapStr{ + + "Binlog_cache_disk_use": toInt(status["Binlog_cache_disk_use"]), + "Binlog_cache_use": toInt(status["Binlog_cache_use"]), + }, + "bytes": common.MapStr{ + + "Bytes_received": toInt(status["Bytes_received"]), + "Bytes_sent": toInt(status["Bytes_sent"]), + }, + "Connections": toInt(status["Connections"]), + "created": common.MapStr{ + + "Created_tmp_disk_tables": toInt(status["Created_tmp_disk_tables"]), + "Created_tmp_files": toInt(status["Created_tmp_files"]), + "Created_tmp_tables": toInt(status["Created_tmp_tables"]), + }, + + "delayed": common.MapStr{ + "Delayed_errors": toInt(status["Delayed_errors"]), + "Delayed_insert_threads": toInt(status["Delayed_insert_threads"]), + "Delayed_writes": toInt(status["Delayed_writes"]), + }, + "Flush_commands": toInt(status["Flush_commands"]), + "Max_used_connections": toInt(status["Max_used_connections"]), + "open": common.MapStr{ + + "Open_files": toInt(status["Open_files"]), + "Open_streams": toInt(status["Open_streams"]), + "Open_tables": toInt(status["Open_tables"]), + "Opened_tables": toInt(status["Opened_tables"]), + }, } return event } + +func toInt(param string) int { + value, err := strconv.Atoi(param) + + if err != nil { + logp.Err("Error converting param to int: %s", param) + value = 0 + } + + return value +} diff --git a/metricbeat/module/mysql/status/fields.yml b/metricbeat/module/mysql/status/fields.yml new file mode 100644 index 000000000000..b00d152269cf --- /dev/null +++ b/metricbeat/module/mysql/status/fields.yml @@ -0,0 +1,39 @@ +mysql-status: + type: group + description: > + MySQL Stats + fields: + - name: mysql-status + type: group + description: > + MySQL Status + fields: + - name: aborted + type: group + description: > + Aborted status fields + fields: + - name: Aborted_clients + type: integer + description: > + The number of connections that were aborted because the client died without closing the connection properly. + + - name: Aborted_connects + type: integer + description: > + The number of failed attempts to connect to the MySQL server. + + - name: bytes + type: group + description: > + Bytes stats + fields: + - name: Bytes_received + type: integer + description: > + The number of bytes received from all clients. + + - name: Bytes_sent + type: integer + description: > + The number of bytes sent to all clients. diff --git a/metricbeat/module/mysql/status/status.go b/metricbeat/module/mysql/status/status.go index bb0bb6784f09..b6f4d88bb4bd 100644 --- a/metricbeat/module/mysql/status/status.go +++ b/metricbeat/module/mysql/status/status.go @@ -1,4 +1,13 @@ -// Fetch status information from mysql: http://dev.mysql.com/doc/refman/5.7/en/show-status.html +/** + +Fetch status information from mysql: http://dev.mysql.com/doc/refman/5.7/en/show-status.html + +TODO @ruflin, 20160315 + * Complete fields read + * Complete template + * Complete dashboards + +*/ package status import ( @@ -71,7 +80,8 @@ func (m *MetricSeter) getConnection(dsn string) (*sql.DB, error) { // loadStatus loads all status entries from the given database into an array func (m *MetricSeter) loadStatus(db *sql.DB) (map[string]string, error) { - rows, err := db.Query("SHOW STATUS") + // Returns the global status, also for versions previous 5.0.2 + rows, err := db.Query("SHOW /*!50002 GLOBAL */ STATUS;") if err != nil { return nil, err } diff --git a/metricbeat/module/mysql/status/status_test.go b/metricbeat/module/mysql/status/status_test.go index 9cec3271308b..1751f9583f60 100644 --- a/metricbeat/module/mysql/status/status_test.go +++ b/metricbeat/module/mysql/status/status_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/module/mysql" ) @@ -28,9 +29,10 @@ func TestFetch(t *testing.T) { // Check event fields connections := event["Connections"].(int) - openTables := event["Open_tables"].(int) - openFiles := event["Open_files"].(int) - openStreams := event["Open_streams"].(int) + open := event["open"].(common.MapStr) + openTables := open["Open_tables"].(int) + openFiles := open["Open_files"].(int) + openStreams := open["Open_streams"].(int) assert.True(t, connections > 0) assert.True(t, openTables > 0) diff --git a/metricbeat/module/redis/.kibana/dashboard/redis.json b/metricbeat/module/redis/.kibana/dashboard/metricbeat-redis.json similarity index 70% rename from metricbeat/module/redis/.kibana/dashboard/redis.json rename to metricbeat/module/redis/.kibana/dashboard/metricbeat-redis.json index bec8721782e4..40ebf0d3ee59 100644 --- a/metricbeat/module/redis/.kibana/dashboard/redis.json +++ b/metricbeat/module/redis/.kibana/dashboard/metricbeat-redis.json @@ -1,13 +1,13 @@ { - "hits": 0, - "timeRestore": false, - "description": "", - "title": "redis", - "uiStateJSON": "{\"P-2\":{\"vis\":{\"legendOpen\":false}},\"P-1\":{\"vis\":{\"legendOpen\":false}}}", - "panelsJSON": "[{\"id\":\"connected_clients\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":1},{\"id\":\"connections-received\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":3}]", - "optionsJSON": "{\"darkTheme\":false}", - "version": 1, + "hits": 0, + "timeRestore": false, + "description": "", + "title": "metricbeat-redis", + "uiStateJSON": "{\"P-2\":{\"vis\":{\"legendOpen\":false}},\"P-1\":{\"vis\":{\"legendOpen\":false}}}", + "panelsJSON": "[{\"id\":\"connected_clients\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":1},{\"id\":\"connections-received\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":3}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" } -} \ No newline at end of file +} diff --git a/metricbeat/module/redis/info/fields.yml b/metricbeat/module/redis/info/fields.yml index 78e070fc5e36..4dc4492228a7 100644 --- a/metricbeat/module/redis/info/fields.yml +++ b/metricbeat/module/redis/info/fields.yml @@ -14,22 +14,22 @@ redis-info: Redis client stats fields: - name: connected_clients - type: int + type: integer description: > Number of client connections (excluding connections from slaves) - name: client_longest_output_list - type: int + type: integer description: > Longest output list among current client connections. - name: client_biggest_input_buf - type: int + type: integer description: > Biggest input buffer among current client connections - name: blocked_clients - type: int + type: integer description: > Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH)