patterns.yml

# 
# PARSER DEFINITIONS FILE IN YML FORMAT
#
# Please use 'ts' as feild name for dates and time
# RegexTools: https://regex101.com/#javascript
# Date format changes in date-fns: 
# DEPRECATED date formats: https://date-fns.org/v2.0.0-alpha.6/docs/parse
# NEW date formats: https://date-fns.org/v2.9.0/docs/parse
#   DD -> dd
#   YYYY -> yyyy
#   ZZ -> X 


# Sensitive data can be replaced with a hashcode (sha1)
# it applies to fields matching the field names by a regular expression
# Note: this function is not optimized (yet) and might cause 10-15% performance hit
#autohash: !!js/regexp /user|client_ip|password|email|credit_card_number|payment_info/i
# set the hash function (default sha256), sha256,sha512
#hashFunction: sha512

debug: false
# set originalLine to false when auothash fields
# the original line might include sensitive data!
originalLine: false
# default seperator for multiline logs, 
# which don't have a blockStart property
# The default /^\S{2,}/ would match typical stack traces
# All lines that start with a whitespace  or contain only one char 
# would be attached to previous lines
multiline:
  defaultSeparator: ^\S{2,}

# post process all JSON input
json:
  enabled: true 
#  autohashFields:
#    _HOSTNAME: true
  debug: false
#  removeFields:
#    - stacktrace
#    - msg
#    - level
#    - time
#  mapFields: 
#    msg: message
#    level: severity
#    time: '@timestamp'
#  transform: !!js/function >
#   function (sourceName, parsedObject, config) {
#    // map fields
#    Object.keys(config.mapFields).forEach(function (f) {
#      if (parsedObject[f] !== undefined) {
#        parsedObject[config.mapFields[f]] = parsedObject[f] 
#        if (config.debug) {
#         console.log('map ' + f + ' to ' + config.mapFields[f] + ': ' + parsedObject[config.mapFields[f]])
#       }
#      }
#    })
#    // remove fields
#    for (var i=0; i<config.removeFields.length; i++) {
#       if (config.debug) {
#         console.log('delete ' + config.removeFields[i])
#       }
#       delete parsedObject[config.removeFields[i]]
#    } 
#   }

# The global transform function is called for each parsed line.
# A good place to do add global fields, custom date processing, or special parsers.
#globalTransform: !!js/function >
#  function (source, parsedObject) {
#    // this function is called after parsing
#    // regardless of the logging source 
#    // for pattern specific functions use transform in pattern definitions
#  }

# IMPORTANT:
# PATTERNS ARE EVALUATED SEQUENTIALLY FOR EACH LOG EVENT. PUT MORE COMPLEX AND MORE SPECIFIC PATTERNS FIRST.
# As soon as a pattern matches a log event the rest of the patterns are skipped.
#
# To test your pattern: cat myTest.log | logagent -n <pattern name here> -yml -f mypatterns.yml
# A pattern name are things like 'kubernetes hyperkube' or 'Elasticsearch' or 'Apache Solr' below.

patterns:
 - # sematext/agent logs 
  sourceName: !!js/regexp /sematext\/agent|containerd|dockerd/
  blockStart: !!js/regexp /^panic\:|^unexpected fault address|^fatal error\:|(^INFO|^ERRO|^WARN|^FAT|^TRAC|^DEB)|^time=|^S{2,}/
  match:
    - type: sematext_agent_golang
      regex: !!js/regexp /time=(\S+)\slevel=(\S+?)\smsg="(.+?)"\ssource="(.+?)"/i
      fields: [ts, severity, message, source]
      dateFormat: iso
    - type: sematext_agent_golang
      regex: !!js/regexp /([A-Z]+)\[(.+?)\]\s(.*)/i
      fields: [severity, ts, message]
      dateFormat: iso
 - # Yandex Clickhouse 
  sourceName: !!js/regexp /clickhouse/
  blockStart: !!js/regexp /(\d{4}.\d{2}.\d{2}[\s|T][\d+|\:]+.\d+)\s\[\s(\d+)\s]\s\{(\S*)\}/
  match:
    - type: clickhouse
      regex: !!js/regexp /(\d{4}.\d{2}.\d{2}[\s|T][\d+|\:]+.\d+)\s\[\s(\d+)\s]\s\{(\S*)\}\s<(\S+)>\s((.+?)\:[\s|\S]+)/i
      fields: [ts, threadNumber, queryId, severity, message, module]
      dateFormat: yyyy.MM.dd HH:mm:ss.SSS
 - # kubernetes hyperkube
  sourceName: !!js/regexp /hyperkube/
  match:
    - type: hyperkube
      regex: !!js/regexp /^\S+\s(\S+)\s+\S+\s+\S+\s([GET|POST|PUT|DELETE|HEAD|OPTIONS]+)\s+(\/.+)\:\s\(([\d|\.]+)(\S+)\)\s(\d+\s)(.*hyperkube.+)\s(.+)\:(\d+)\]/i
      fields: [ts,method,url,duration,duration_unit,status_code,info,ip,port]
      dateFormat: HH:mm:ss:S
 - # Elasticsearch 
  blockStart: !!js/regexp /^[\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+|log4j\:\S+\s/
  sourceName: !!js/regexp /elasticsearch/
  match:
    - type: elasticsearch_slow_log
      regex: !!js/regexp /^\[(\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+)\]\[(.+?)\s*\]\[(\S{0,512})\s*\]\s*\[(.+?)\]\s\[(\S+?)\]\[(\d+)\]\s.+took_millis\[(\d+)\].+types\[(.*?)\].+stats\[(.*?)\].*search_type\[(.*?)\].+total_shards\[(.*?)\].+source\[(.*?)\],/i
      fields: 
        - ts
        - severity:string
        - class_name:string
        - node_name:string
        - index_name:string
        - shard_number:number
        - took_millis:number
        - types:string
        - stats:string
        - search_type:string
        - total_shards:number
        - source:string
    - type: elasticsearch
      regex: !!js/regexp /^\[(\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+)\]\[(.+?)\s*\]\[(\S{0,512})\s*\]\s*\[(.+?)\]\s([\s|\S]+)/
      fields: [ts,severity,class_name,node_name,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: elasticsearch
      regex: !!js/regexp ^\[(\d{4}-\d{2}-\d{2}\s[\d+|\:]+.\d+)\]\[(.+?)\]\[(\S{0,512})\s*\]\s*\s([\s|\S]+)
      fields: [ts,severity,class_name,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS

 - # Apache Solr
  blockStart: !!js/regexp ^\S*\s*\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3}|^\d+\s+\S{3,5}\s+
  sourceName: !!js/regexp /solr/i
  match:
    - type: apache_solr_7_8_hits
      regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s\[(.+?)\]\s+webapp=(.+?)\spath=(.+?)\sparams={(.*)}\shits=(\d+)\sstatus=(\d+)\sQTime=(\d+)
      fields: 
        - ts
        - severity:string
        - thread:string
        - core:string
        - class:string
        - shard:string
        - webapp:string
        - path:string
        - params:string
        - hits:number
        - status:number
        - qtime:number
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
      transform: !!js/function >
        function (p) {
          if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')
          {
            var params = p.params.split('&')
            p.parsedParams={}
            for(var i=0;i<params.length;i++)
            {
              var key_value = params[i].split('=')
              if (key_value.length>1) {
                key = key_value[0]
                if (!/[a-zA-Z]/.test(key)) {
                  key = "field_" + key
                }

                if (key in p.parsedParams) {
                  // convert from single to multivalued
                  if (typeof p.parsedParams[key] == 'string') {
                    p.parsedParams[key] = [p.parsedParams[key]]
                  }

                  p.parsedParams[key].push(key_value[1])
                } else {
                  p.parsedParams[key] = key_value[1]
                }
              }
            }
            if (p.parsedParams['NOW']) {
              p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)
              // if (!p['@timestamp'])
              //  p['@timestamp'] = p.paramsParsed['NOW']
            }
          }

          var core = p.core.split(' ')

          //sometimes core=0, sometimes core="c:test s:shard2...."
          if (core.length>1){
            p.parsedCore={}
            for(var i=0;i<core.length;i++)
            {
              var key_value = core[i].split(':')
              if (key_value.length>1) {
                p.parsedCore[key_value[0]] = key_value[1]
              }
            }
          }
        }
    - type: apache_solr_7_8
      regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s\[(.+?)\]\s+webapp=(.+?)\spath=(.+?)\sparams={(.*)}\sstatus=(\d+)\sQTime=(\d+)
      fields: 
        - ts
        - severity:string
        - thread:string
        - core:string
        - class:string
        - shard:string
        - webapp:string
        - path:string
        - params:string
        - status:number
        - qtime:number
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
      transform: !!js/function >
        function (p) {
          if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')
          {
            var params = p.params.split('&')
            p.parsedParams={}
            for(var i=0;i<params.length;i++)
            {
              var key_value = params[i].split('=')
              if (key_value.length>1) {
                key = key_value[0]
                if (!/[a-zA-Z]/.test(key)) {
                  key = "field_" + key
                }

                if (key in p.parsedParams) {
                  // convert from single to multivalued
                  if (typeof p.parsedParams[key] == 'string') {
                    p.parsedParams[key] = [p.parsedParams[key]]
                  }

                  p.parsedParams[key].push(key_value[1])
                } else {
                  p.parsedParams[key] = key_value[1]
                }
              }
            }
            if (p.parsedParams['NOW']) {
              p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)
              // if (!p['@timestamp'])
              //  p['@timestamp'] = p.paramsParsed['NOW']
            }
          }

          var core = p.core.split(' ')

          //sometimes core=0, sometimes core="c:test s:shard2...."
          if (core.length>1){
            p.parsedCore={}
            for(var i=0;i<core.length;i++)
            {
              var key_value = core[i].split(':')
              if (key_value.length>1) {
                p.parsedCore[key_value[0]] = key_value[1]
              }
            }
          }
        }
    - type: apache_solr_audit_log
      regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s*\((.+?)\)\s\[(.+?)]\s(.+?)\stype="(.*?)"\smessage="(.*?)"\smethod="(.*?)"\sstatus="(.*?)"\srequestType="(.*?)"\susername="(.*?)"\sresource="(.*?)"\squeryString="(.*?)"\scollections=(.*)
      fields: 
        - ts
        - severity:string
        - thread:string
        - core:string
        - class:string
        - type:string
        - message:string
        - method:string
        - status:number
        - requestType:string
        - username:string
        - resource:string
        - querystring:string
        - collections:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: apache_solr_7_8_generic
      regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s(.*)
      fields: 
        - ts
        - severity:string
        - thread:string,
        - core:string
        - class:string
        - message:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
      transform: !!js/function >
        function (p) {
          var core = p.core.split(' ')

          //sometimes core=0, sometimes core="c:test s:shard2...."
          if (core.length>1){
            p.parsedCore={}
            for(var i=0;i<core.length;i++)
            {
              var key_value = core[i].split(':')
              if (key_value.length>1) {
                p.parsedCore[key_value[0]] = key_value[1]
              }
            }
          }
        }
    - type: apache_solr_v4.6
      regex: !!js/regexp ^(\S+)\s+-\s(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s(.+?);\s\[(.+?)]\swebapp=(\S+)\spath=(.+?)\sparams={(.*)}.*hits=(\d+)\sstatus=(\d+)\sQTime=(\d+)
      fields: [severity,ts,class,application,webapp,path,params,hits,status,qtime]
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
      transform: !!js/function >
        function (p) {
          if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')
          {
            var params = p.params.split('&')
            p.parsedParams={}
            for(var i=0;i<params.length;i++)
            {
              var key_value = params[i].split('=')
              if (key_value.length>1) {
                p.parsedParams[key_value[0]] = key_value[1]
              }
            }
            if (p.parsedParams['NOW']) {
              p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)
              // if (!p['@timestamp'])
              //  p['@timestamp'] = p.paramsParsed['NOW']
            }
          }
        }
    - type: apache_solr
      regex: !!js/regexp ^(\S+)\s+-\s(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s\[\s*(.+?)]\s(\S+);\s.*.*webapp=(\S+)\spath=(.+?)\sparams={(.*)}.*hits=(\d+)\sstatus=(\d+)\sQTime=(\d+)
      fields: [severity,ts,application,class,webapp,path,params,hits,status,qtime]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_solr_v5_1
      regex: !!js/regexp ^(\d+)\s\[(\S+)]\s(\S+)\s(\S+)\s\[(\S+)\s(\S+)\s(\S+)\s(\S+)\].+?\[(.+?)\]\swebapp=(.+?)\spath=(.+?)\sparams={(.+?)}\sstatus=(\d+)\sQTime=(\d+)
      fields: [relative_ts,thread_id,severity,class,collection,shard,core,replica,core_name,webapp,path,params,status,qtime]
      transform: !!js/function >
        function (p) {
          if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')
          {
            var params = p.params.split('&')
            p.parsedParams={}
            for(var i=0;i<params.length;i++)
            {
              var key_value = params[i].split('=')
              if (key_value.length>1)
                p.parsedParams[key_value[0]] = key_value[1]
            }
            if (p.parsedParams['NOW']) {
              p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)
              // if (!p['@timestamp'])
              //   p['@timestamp'] = p.paramsParsed['NOW']
            }
          }
        }
    - type: apache_solr
      regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s+(\S+);\s+(.+Exception:\s[\s|\S]+)
      fields: [severity,ts,class,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_solr
      regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s+(\S+);\s([\s|\S]+)
      fields: [severity,ts,class,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_solr_5_generic
      regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s(.*)
      fields: [severity,ts,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_solr4
      regex: !!js/regexp ^(\d+)\s+(\S+)\s+\((\S+)\)\s+\[(.+?)\]\s(\S+)\s(.+)
      fields: [relative_ts,severity,thread,thread_id,class,message]

 - # Apache Kafka
  sourceName: !!js/regexp /kafka/
  match:
    - type: apache_kafka
      regex: !!js/regexp ^\[(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\]\s(\S+)\s(.+)
      fields: [ts,severity,message]
      dateFormat: yyyy-MM-dd HH:mm:ss

 - # Apache HDFS Data Node
  blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s(\S+)\s/
  sourceName: !!js/regexp /hdfs/
  match:
    - type: apache_hdfs_data_node
      regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s+[\d|\:]+,\d+)\s+(\S+)\s(\S+):\s([\s|\S]+)
      fields: [ts,severity,class,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS

 - # Apache HBase Region Server
  blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s(\S+)\s/
  sourceName: !!js/regexp /hbase/
  match:
    - type: apache_hbase_region_server
      regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s+[\d|\:]+,\d+)\s+(\S+)\s+\[(.+)\]\s(\S+):\s([\s|\S]+)
      fields: [ts,severity,thread,class,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS

 - # Apache YARN
  sourceName: !!js/regexp /yarn/
  match:
    - type: apache_hadoop_yarn_node_manager
      regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s(\S+)\s(\S+):\s([\S|\s]+)
      fields: [ts,severity,class_name,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS

 - # Apache Zookeeper
  sourceName: !!js/regexp /zookeeper|zk/
  blockStart: !!js/regexp /^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s+/
  match:
    - type: apache_zookeeper
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+[client|from]\s\/(.+?)\:(\d+).*sessionid.+(0x.+).*)/
      fields: 
        - ts
        - machineId:string
        - severity:string,
        - thread_info:string
        - message:string
        - client_ip:string
        - client_port:number
        - session_id:string
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_zookeeper
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+[client|from]\s\/(.+?)\:(\d+).*)/
      fields: 
        - ts
        - machineId:string
        - severity:string,
        - thread_info:string
        - message:string
        - client_ip:string
        - client_port:number
      dateFormat: yyyy-MM-dd HH:mm:ss,SS
    - type: apache_zookeeper
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+)/
      fields: 
        - ts
        - machineId:string
        - severity:string,
        - thread_info:string
        - message:string
      dateFormat: yyyy-MM-dd HH:mm:ss,SS


 - # Apache Cassandra
  sourceName: !!js/regexp cassandra
  # multi-line, start sequence
  blockStart: !!js/regexp ^\S{3,5}\s+\[.+\]\s+\d{4}
  match:
    - type: apache_cassandra
      regex: !!js/regexp ^\S{0,5}(\S*)\s+\[(.+)\]\s(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+(.+\.java)\:(\d+)\s+-\s+([\S|\s]+)
      fields: [severity,module,ts,java_file,code_line,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SS

 - # MongoDB
  # name of the docker image
  sourceName: !!js/regexp /mongo/
  # 2015-07-28T00:35:46.329+0000 I JOURNAL  [initandlisten] journal dir=/data/db/journal
  match: 
    - type: mongodb
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}T[\d|\.|\:]+\+\d{4})\s(\w+)\s(\S+)\s+\[(\S+)\]\s(.+)/i
      fields:  [ts,severity, component, context, message]
      dateFormat: iso

 - # REDIS
  # name of the docker image
  # example: "1:M 22 Jul 21:58:28.146 # Server started, Redis version 3.0.2"
  sourceName: !!js/regexp /redis/i
  blockStart: !!js/regexp /^\d+:.\s\d+/
  match:
  - type: redis
    blockStart: !!js/regexp /^\d+:.\s\d+/
    fields: [pid,role,ts,level,message]
    regex: !!js/regexp /^(\d+):(\w+)\s+(\d\d\s\w+.+)\s+(\S)\s+([\S|\s]+)/
    dateFormat: dd MMM HH:mm:ss.SSS
    transform: !!js/function >
      function transformMessage (p) {
        levels = {
          '.': 'debug',
          '-': 'verbose',
          '*': 'notice',
          '#': 'warning'
        }
        roles = {
          'X': 'sentinel',
          'C': 'RDB/AOF writing child',
          'S': 'slave',
          'M': 'master'
        }
        p.role = roles[p.role]
        p.severity = levels[p.level] || p.level
        // p.message = p.message.replace(/\r/g,'')
        delete p.level 
      } 

 - # Sonatype Nexus
  sourceName: !!js/regexp /nexus/
  # yyyy-MM-dd starts a new log entry
  blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}
  match:
    - type: nexus
      regex: !!js/regexp /^([\d|\-|\s|\:|\.|\,|\+]+)\s+([A-Z]+)\s+[^\[]*\[\s*([^\]]+)\]\s(\*?\w+)\s+([\w|\.]+)\W+(.+)/
      fields: [ts,severity,thread,user,class,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,SSSX

 - # NodeBB Forum
  sourceName: !!js/regexp /nodebb/i
  match:
  - type: nodebb_forum
    fields: [ts,severity,module,message]
    regex: !!js/regexp /^(\d{4}\-\d{2}\-\d{1,2}T\d\d:\d\d:\d\d\.\d+Z)\s-\s(\w+):\s\[(\S+)]\s(.*)/
    dateFormat: iso
  - type: nodebb_forum
    fields: [ts,severity,message]
    regex: !!js/regexp /^(\d{4}\-\d{2}\-\d{1,2}T\d\d:\d\d:\d\d\.\d+Z)\s-\s(\w+):\s(.*)/
 - # mysql
  # 2015-07-25 14:11:35 0 [Note] mysqld (mysqld 5.6.26) starting as process 1 ...
  sourceName: !!js/regexp /mysql/ 
  match: 
    - regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+)\s(\d+)\s\[(.+?)\]\s+(.*)/
      fields: [ts,pid,level,message]
      dateFormat: yyyy-MM-dd HH:mm:ss
      type: mysql

 - # nsq.io  
  sourceName: !!js/regexp nsqio\/nsq
  match:
    - type: nsq
      regex: !!js/regexp (^\d{4}\/\d{2}\/\d{2}\s[\d|\:]+)\s(\S+)\s+(\d+)\s+\[(\S+)\]\s+(.+)
      fields: [ts, level, pid, module, message]
      dateFormat: yyyy/MM/dd HH:mm:ss

 - #  Web Logs
  sourceName: !!js/regexp /httpd|access_log|apache2|nginx|sematext\/frontend-app/
  match: 
    - type: access_log_combined
      regex: !!js/regexp ^([0-9a-f.:]+)\s(-|\S+)\s(-|\S+)\s\[(.*)\]\s\"(\w+)\s(\S+)\s{0,1}(.*)\" ([0-9|\-]+) ([0-9|\-]+) \"([^\"]+)\" \"([^\"]+)\"
      fields: 
        - client_ip:string
        - remote_id:string
        - user:string
        - ts
        - method:string
        - path:string
        - protocol:string
        - status_code:number
        - size:number
        - referer:string
        - user_agent:string
      dateFormat: dd/MMM/yyyy:HH:mm:ss X
      transform: !!js/function >
        function transformMessage (p) {
          p.message = p.method + ' ' + p.path
          if(p.status_code === '-') {
            p.status_code = 0
          }
          if(p.size === '-') {
            p.size = 0
          }
        } 
    # nginx proxy jwilder/nginx-proxy
    - regex: !!js/regexp /^(\S+)\s+(-|.+?)\s+(-|.+?)\s+\[(.*)\]\s\"(\S+)\s(\S+)\s(\S+)\"\s(\d+)\s(\d+|\"-\"|-)\s+"{0,1}(.+?)"{0,1}\s+"{0,1}([\S|\s]+)"{0,1}/i
      type: access_common
      fields: 
        - proxy_service:string
        - virtual_host:string
        - client_ip:string
        - remote_id:string
        - user:String
        - ts
        - method:string
        - path:string
        - http_version:string
        - status_code:number
        - size:number
        - url:string
        - user_agent:string
      dateFormat: dd/MMM/yyyy:HH:mm:ss X
      #transform: !!js/function >
      #  function transformMessage (p) {
      #    p.message = p.method + ' ' + p.path
      #    if(p.status_code === '-') {
      #      p.status_code = 0
      #    }
      #    if(p.size === '-') {
      #      p.size = 0
      #    }
      #  } 
    - regex: !!js/regexp ^(\S+)\s+(-|.+?)\s+(-|.+?)\s+\[(.*)\]\s\"(\S+)\s(\S+)\s(\S+)\"\s(\d+)\s(\d+|\"-\"|-)
      type: access_common
      fields: 
        - client_ip:string
        - remote_id:string
        - user:string
        - ts
        - method:string
        - path:string
        - http_version:string
        - status_code:number
        - size:number
      dateFormat: dd/MMM/yyyy:HH:mm:ss X
      #transform: !!js/function >
      #  function transformMessage (p) {
      #    p.message = p.method + ' ' + p.path
      #    if(p.status_code === '-') {
      #      p.status_code = 0
      #    }
      #    if(p.size === '-') {
      #      p.size = 0
      #    }
      #  } 
    - type: nginx_error_log 
      regex: !!js/regexp /^(\d{4}\/\d{2}\/\d{2}\s[\d|\:]+)\s\[(.+?)]\s(\d+)#(\d+)\:\s(.*)/
      fields: [ts,level,pid,tid,message]
      dateformat: yyyy/MM/dd HH:mm:ss

    - type: apache_error_log
      regex: !!js/regexp /^\[(\w{3} \w{3} \d{2} [\d|\:]+\s\d+)\] \[(.+?)\] \[client ([\d|\.]+)\] (.+)/
      fields: [ts,level,client_ip,message]
      dateformat: ddd MMM dd hh:mm:ss.SSS yyyy
       
    # Apache MPM events
    - regex: !!js/regexp /^\[(.+?)\]\s+\[(.+?)\]\s+\[(.+?)\]\s+(.+)/
      fields: [ts,event_type,processInfo,message]
      type: apache_mpm
      dateformat: ddd MMM dd hh:mm:ss.SSS yyyy

 - # Apache Flink jobmanager and taskmanager
  sourceName: !!js/regexp /flink/
  blockStart: !!js/regexp /^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3}\s/
  match:
    - type: flink[job|task]manager
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3})\s(INFO|info|WARN|ERROR|FATAL)\s+([\S]+)\s+-\s([\S|\s|\n]+)/
      fields: [ts, severity, processid, message]
      dateFormat: yyyy-MM-dd HH:mm:ss:l
 - # Apache Flink jobmanager and taskmanager
  sourceName: !!js/regexp /flink/
  blockStart: !!js/regexp /^\w{3}\s[\d|,|\s|\:]+/
  match:
    - type: flink[job|task]manager
      regex: !!js/regexp /^(\w{3}\s[\d|,|\s|\:]+\w{3}\s[\d|,|\s|\:]+[AP]M)\s([\S|\s]+)\s(INFO|WARN|ERROR|SEVERE|FATAL):\s([\S|\s|\n]+)/
      fields: [ts, processid, severity, message]
      dateFormat: MMM dd, yyyy H:mm:ss

 - # Traefik access_log
  sourceName: !!js/regexp /traefik/
  match:
    - type: traefik_access_log
      regex: !!js/regexp ^([0-9a-f.:]+)\s(-|\S+)\s(-|\S+)\s\[(.*)\]\s\"(\w+)\s(\S+)\s{0,1}(.*)\"\s([0-9|\-]+)\s([0-9|\-]+)\s\"([^\"]+)\"\s\"([^\"]+)\"\s([0-9|\-]+)\s\"(.+)\"\s\"(.+)\"\s([0-9]+)ms
      fields: 
        - client_ip:string
        - remote_id:string
        - user:string
        - ts
        - method:string
        - path:string
        - protocol:string
        - status_code:number
        - size:number
        - referer:string
        - user_agent:string
        - req_count:string # https://github.com/containous/traefik/blob/master/middlewares/accesslog/logger_formatters.go#L45
        - frontend_name:string
        - backend_url:string
        - response_time:number
      dateFormat: dd/MMM/yyyy:HH:mm:ss X
      transform: !!js/function >
        function transformMessage (p) {
          p.message = p.method + ' ' + p.path
          if(p.status_code === '-') {
            p.status_code = 0
          }
          if(p.size === '-') {
            p.size = 0
          }
        }

 - # Tutum Logs
  sourceName: !!js/regexp /tutum\/cleanup/
  match:
    - type: tutum_cleanup
      regex: !!js/regexp /^(\d+\/\d+\/\d+\s\d+:\d+:\d+)\s(.*)/
      fields: [ts,message]
      dateFormat: yyyy/MM/dd hh:mm:ss

 - # RabbitMQ
  sourceName: !!js/regexp /rabbitmq/
  blockStart: !!js/regexp /^\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d/
  match:
    - type: rabbitmq_startup
      # regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s*\n\s*(Starting\sRabbitMQ\s(\S+)\son\sErlang\s(.+)\n[\s\S]*)/
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s*\n\s*(Starting\sRabbitMQ\s(\S+)\son\sErlang\s(.+)\s*\n\s*(Copyright\s\(c\)\s\d{4}-\d{4}\s[^\n]*)\s*\n\s*Licensed\sunder\sthe\s(\S*\s?\d*\.?\d?)[\s\S]*)/
      fields:
        - ts
        - severity
        - connection:string
        - message
        - rabbitmq_version
        - erlang_version
        - copyright
        - license
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_wal_recovery
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(WAL:\srecovering\s\[(.*)\])/
      fields:
       - ts
       - severity
       - connection:string
       - message
       - wal_path:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_access_denied
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(.+access\sdenied:\suser\s'(\S+)'\s-\s(.+))/
      fields:
       - ts
       - severity
       - connection:string
       - message
       - user:string
       - reason:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_login_refused
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\sError\son\sAMQP\sconnection\s<(\S+)>\s\((\S+):(\d+)\s->\s(\S+):(\d+),\sstate:\s(\S+)\):\n((\S+)\slogin\srefused:\suser\s'(\S+)'\s-\s(.+))/
      fields:
       - ts
       - severity
       - connection:string
       - refused_connection:string
       - source_address:string
       - source_port:number
       - destination_address:string
       - destination_port:number
       - state:string
       - message
       - auth_method:string
       - user:string
       - reason:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_connections_authenticated
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+)\):\s(user\s'(\S+)'\sauthenticated\sand\sgranted\saccess\sto\svhost\s'(\S+)\')/
      fields:
       - ts
       - severity
       - connection:string
       - source_address:string
       - source_port:number
       - destination_address:string
       - destination_port:number
       - message
       - user:string
       - vhost:string
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_connections_vhost_user
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(accepting|closing)\sAMQP\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+),\svhost:\s'(.+)',\suser:\s'(.+)'\):?([\s\S]*)/
      fields:
       - ts
       - severity
       - connection:string
       - connection_event
       - source_address:string
       - source_port:number
       - destination_address:string
       - destination_port:number
       - vhost:string
       - user:string
       - message
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq_connections
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(accepting|closing)\sAMQP\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+)\)([\s\S]*)/
      fields:
       - ts
       - severity
       - connection:string
       - connection_event
       - source_address:string
       - source_port:number
       - destination_address:string
       - destination_port:number
       - message
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS
    - type: rabbitmq
      regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s\<(\S+)\>\s([\s\S]+)/
      fields:
       - ts
       - severity
       - connection:string
       - message
      dateFormat: yyyy-MM-dd HH:mm:ss.SSS

 - # Postgres
  sourceName: !!js/regexp /postgres/
  blockStart: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]/
  match:
   - type: postgres_slowlog
     regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+)@(\S+)\s(\S+):\s+duration:\s(\S+)\sms\s+(\S+).*:\s+([\S|\s]+)/
     fields:
      - ts
      - timezone:string
      - pid:number
      - user:string
      - database:string
      - severity:string
      - duration_ms:number
      - operation:string
      - statement:string
     dateFormat: yyyy-MM-dd HH:mm:ss.SSS
   - type: postgres_with_user
     regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+)@(\S+)\s(\S+):\s+([\S|\s]+)/
     fields:
      - ts
      - timezone:string # assuming timezone is always "UTC", but parsing it just in case
      - pid:number
      - user:string
      - database:string
      - severity:string
      - message
     dateFormat: yyyy-MM-dd HH:mm:ss.SSS
   - type: postgres
     regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+):\s+([\S|\s]+)/
     fields:
      - ts
      - timezone:string # assuming timezone is always "UTC", but parsing it just in case
      - pid:number
      - severity:string
      - message
     dateFormat: yyyy-MM-dd HH:mm:ss.SSS


 - # CouchDB
  sourceName: !!js/regexp /couchdb/
  match:
   - type: couchdb
     regex: !!js/regexp /^\[(\S+)\]\s(\S+)\s(\S+)\s(\S+)\s(\S{8})\s([\s|\S]+)/
     fields: 
      - severity:string
      - ts
      - node:string
      - module:string
      - code:string
      - message:string
     dateFormat: iso
     transform: !!js/function >
        function (p) {
          p.os = {host: p.node}
        }
   - type: couchdb_http
     regex: !!js/regexp /^\[(\S+)\]\s(\S+)\s(\S+)\s(\S+)\s(\S{10})\s(\S+):(\d+)\s(\S+)\s(\S+)\s(GET|PUT|POST|DELETE|HEAD)\s(\S+)\s(\d+)\s(\S|\s+)/
     fields: 
      - severity:string
      - ts
      - node:string
      - module:string
      - code:string
      - server_ip:string
      - server_port:number
      - client_ip:string
      - user:string
      - method:string
      - url:string
      - status_code:number
      - response:string
     transform: !!js/function >
        function (p) {
          p.os = {host: p.node}
        }
     dateFormat: iso

 - # Heroku Syslog Messages
  sourceName: !!js/regexp /syslog_framed|heroku/ 
  match:
    -
      type: heroku
      # blockStart: !!js/regexp \^+\s<(\d+)>(\d+)\s/
      regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s(\S+)\s(\S+)\s(\S+)\s(\S+)\.{0,1}(\d*)\s+-\s(.*)
      fields: [prio,version,ts,host,app,process_type,dyno,message]
      dateFormat: iso
      transform: !!js/function >
        function (p) {
          p.os = {host: p.host}
          const SEVERITY = [
            'emerg',
            'alert',
            'crit',
            'err',
            'warning',
            'notice',
            'info',
            'debug'
          ]
          const FACILITY = [
            'kern',
            'user',
            'mail',
            'daemon',
            'auth',
            'syslog',
            'lpr',
            'news',
            'uucp',
            'cron',
            'authpriv',
            'ftp',
            'ntp',
            'logaudit',
            'logalert',
            'clock',
            'local0',
            'local1',
            'local2',
            'local3',
            'local4',
            'local5',
            'local6',
            'local7'
          ]
          p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)
          p.severity = SEVERITY[p.prio&7] || String(p.prio&7)
          if (p.process_type === 'router')
          {
            try { 
            var keyValue = p.message.trim().split(' ')
            keyValue.forEach (function (kv) {
              var kvs = kv.split ('=')
              p[kvs[0].trim()] = kvs[1].trim()
            })
            } catch (err) {
              // ignore
            }
          }
        }

 - # CloudFoundry Syslog Messages
  sourceName: !!js/regexp /cloudfoundry.*|syslog_raw/ 
  match:
    - 
      type: cloudfoundry
      regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s([\d|-]+T[\d|\:|.|\+]+)\s([a-zA-Z0-9\-\.\_\s]+)\s(.+?)\s\[(.+)\]\s-\s\[(.*)\]\s([a-zA-Z0-9\.\_]+)\s-\s\[[\d|-]+T[\d|\:|.]+Z\]\s(.*)
      fields: [prio,version,ts,host,processID,applicationID,tags,domain,message]
      dateFormat: iso
      transform: !!js/function >
        function (p) {
          const SEVERITY = [
            'emerg',
            'alert',
            'crit',
            'err',
            'warning',
            'notice',
            'info',
            'debug'
          ]
          const FACILITY = [
            'kern',
            'user',
            'mail',
            'daemon',
            'auth',
            'syslog',
            'lpr',
            'news',
            'uucp',
            'cron',
            'authpriv',
            'ftp',
            'ntp',
            'logaudit',
            'logalert',
            'clock',
            'local0',
            'local1',
            'local2',
            'local3',
            'local4',
            'local5',
            'local6',
            'local7'
          ]
          p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)
          p.severity = SEVERITY[p.prio&7] || String(p.prio&7)
          delete p.prio
          p.os = {host: p.host}
        }
    - 
      type: cloudfoundry
      regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s([\d|-]+T[\d|\:|.|\+]+)\s([a-zA-Z0-9\-\.\_\s]+)\s(.+?)\s\[(.+)\]\s-\s\[(.*)\]\s(.*)
      fields: [prio,version,ts,host,processID,applicationID,tags,message]
      dateFormat: iso
      transform: !!js/function >
        function (p) {
          const SEVERITY = [
            'emerg',
            'alert',
            'crit',
            'err',
            'warning',
            'notice',
            'info',
            'debug'
          ]
          const FACILITY = [
            'kern',
            'user',
            'mail',
            'daemon',
            'auth',
            'syslog',
            'lpr',
            'news',
            'uucp',
            'cron',
            'authpriv',
            'ftp',
            'ntp',
            'logaudit',
            'logalert',
            'clock',
            'local0',
            'local1',
            'local2',
            'local3',
            'local4',
            'local5',
            'local6',
            'local7'
          ]
          p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)
          p.severity = SEVERITY[p.prio&7] || String(p.prio&7)
          delete p.prio
          p.os = {host: p.host}
        }
    - 
      type: cloudfoundry
      regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s([\d|-]+T[\d|\:|.|\+]+)\s(\S+)\s(.+?)\s\[(.+)\]\s-\s-\s(.+)
      fields: [prio,version,ts,host,applicationID,processID,message]
      dateFormat: iso
      transform: !!js/function >
        function (p) {
          const SEVERITY = [
            'emerg',
            'alert',
            'crit',
            'err',
            'warning',
            'notice',
            'info',
            'debug'
          ]
          const FACILITY = [
            'kern',
            'user',
            'mail',
            'daemon',
            'auth',
            'syslog',
            'lpr',
            'news',
            'uucp',
            'cron',
            'authpriv',
            'ftp',
            'ntp',
            'logaudit',
            'logalert',
            'clock',
            'local0',
            'local1',
            'local2',
            'local3',
            'local4',
            'local5',
            'local6',
            'local7'
          ]
          p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)
          p.severity = SEVERITY[p.prio&7] || String(p.prio&7)
          delete p.prio
          p.os = {host: p.host}
        }

 - # Docker Swarm 
  sourceName: !!js/regexp /swarm/ # catch all .log files  
  match:
    -
      type: docker
      regex: !!js/regexp /^time="(.*)\slevel=(\S+)\smsg="(.+?)"\saddr="(.+?)"\sdiscovery="(.+?)"/
      fields: [ts,severity,message,address,discovery]
    -
      type: docker
      regex: !!js/regexp /^time="(.*)\slevel=(\S+)\smsg="(.+?)/
      fields: [ts,severity,message]
   
 - # timestamped messages from /var/log/*.log on Mac OS X
  sourceName: !!js/regexp /\.log/ # catch all .log files  
  match:
    -
      type: system_log
      regex: !!js/regexp /^([\w|\s]+\s+\d{1,2}\s[\d|\:|\.]+)\s+(\S+)\s+(.*)\.(.+)\s(.+?)\[(\d+)\]\:\s+(.*)/
      fields: [ts,syslog_host,syslog_facility,severity,syslog_service,syslog_pid,message]
      dateFormat: MMM dd HH:mm:ss
    -
      type: system_log
      regex: !!js/regexp /^([\w|\s]+\s+\d{2}\s[\d|\:]+)\s(.+?)\s(.+?)\s<(.+)>(.*)/
      fields: [ts,syslog_host,service,severity,message]
      dateFormat: MMM dd HH:mm:ss
    -
      type: system_log
      regex: !!js/regexp /^([\w|\s]+\s+\d{1,2}\s[\d|\:]+)\s(\S+)\s(\S+)\[(\d+)\]\s{0,4}<(.+)>\:\s{0,2}(.+)/
      fields: [ts,syslog_host,service,pid,severity,message]
      dateFormat: MMM dd HH:mm:ss
    -
      type: system_log
      regex: !!js/regexp /^([\w|\s]+\s+\d{1,2}\s[\d|\:]+)\s(\S+)\s(\S+)\[(\d+)\]\:\s(.+)/
      fields: [ts,host_syslog,service,pid,message]
      dateFormat: MMM dd HH:mm:ss
    -
      type: system_log
      regex: !!js/regexp /^([\w|\s]+\s+\d{1,2}\s[\d|\:|\.]+)\s+(\S+)\s+(\S+)\:\s(.*)/
      fields: [ts,host_syslog,service,message]
      dateFormat: MMM dd HH:mm:ss
    - 
      type: log
      regex: !!js/regexp /^([\w|\s]+\s\d{2}\s[\d|\:|\.]+)\s+(<.+?>)\s(.*)/
      fields: [ts,service,message]
      dateFormat: MMM D HH:mm:ss
    - 
      type: log
      regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d+)\s\[(\S+)]\s(.+)/
      fields: [ts,severity,message]
      dateFormat: yyyy-MM-dd HH:mm:ss,S
    - 
      type: log
      regex: !!js/regexp /^(\d{4}[\-|\d{2}]+\s[\d|\:]+\s\+\d{4})\:\s+(.+)/
      fields: [ts,message]
      dateFormat: yyyy-MM-dd HH:mm:ss X

 - # Logagent-js ISO timestamp + message
  sourceName: !!js/regexp /logagent/   
  match:    
    -
      type: logagent-js
      regex: !!js/regexp /^(\[\d|\:|\-]+Z)\s([\S|\s]+)/
      fields: [ts,message]
      # see: https://date-fns.org/v2.9.0/docs/parseISO
      dateFormat: iso

dateFormats: [
    'iso',
    'dd/MMM/yyyy:HH:mm:ss X', #apache
    'MMM D HH:mm:ss',
    'MMM dd HH:mm:ss',
    'dd MMM HH:mm:ss.S',
    'dd MMM HH:mm:ss',
    'ddD MMM dd HH:mm:ss',
    'yyyy-MM-dd',
    'yyyy-MM-dd HH:mm',
    'yyyy-MM-dd HHmm',
    'yyyyMMdd HH:mm',
    'yyyyMMdd HHmm',
    'yyyyMMdd',
    'yyyy-MM-dd HH:mm:ss',
    'yyyy-MM-dd HHmmss',
    'yyyy-MM-dd HH:mmX',
    'yyyy-MM-dd HHmmX',
    'yyyy-MM-dd HH:mm:ssX',
    'yyyy-MM-dd HHmmssX',
    'yyyyMMdd HH:mmX',
    'yyyyMMdd HHmmX',
]