Add fluentd configuration for MySQL error.log

- changed default logging to log file
- parametrized log file path
- added fluentd configuration
- introduced fluentd multi_format parser plugin

Currently error logs sent to syslog are represented
with severity ERROR, which is misleading for Notice
and Warning messages during service startup/shutdown.

Change-Id: I55106775fa54b412b50ecc10c12930445da15d03
Related-bug: PROD-21904 (PROD:21904)

README.rst

 - ``galera_xtrabackup_parallel``
    Default is 4
 - ``galera_error_log_enabled``
-   Default is ``false``
+   Default is ``true``
+- ``galera_error_log_path``
+   Default is ``/var/log/mysql/error.log``
 
 Usage:
 
       galera_wsrep_slave_threads: 8
       galera_xtrabackup_parallel: 2
       galera_error_log_enabled: true
+      galera_error_log_path: /var/log/mysql/error.log
 
 Usage
 =====

galera/files/my.cnf

 {% endif %}
 
 [mysqld_safe]
+{%- if not service.get('error_log_enabled', False) %}
 syslog
+{%- endif %}
 
 [mysqld]
 datadir=/var/lib/mysql
 character-set-server=utf8
 default-storage-engine=innodb
 {%- if service.get('error_log_enabled', False) %}
-log_error = /var/log/mysql/error.log
-{%- else %}
-#log_error = /var/log/mysql/error.log
+log_error = {{ service.get('error_log_path', '/var/log/mysql/error.log') }}
 {%- endif %}
 skip-external-locking
 skip-name-resolve

galera/meta/fluentd.yml

+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+agent:
+  config:
+    label:
+      mysql:
+        input:
+          tail_mysql:
+            type: tail
+            tag: mysql.log
+            path: {{ service.get('error_log_path', '/var/log/mysql/error.log') }}
+            pos_file: {{ pillar.fluentd.agent.dir.positiondb }}/mysql.pos
+            parser:
+              type: multi_format
+              patterns:
+                # mysql format I: https://regex101.com/r/xmjZ8H/1
+                - type: regexp
+                  time_key: Timestamp
+                  time_format: '%F %H:%M:%S'
+                  keep_time_key: false
+                  format: '/^(?<Timestamp>.*) (?<Pid>\d+) \[(?<Severity>.*)\] (?<Payload>.*)$/'
+                # mysql format II: https://regex101.com/r/ucqhB3/1
+                - type: regexp
+                  time_key: Timestamp
+                  time_format: '%H:%M:%S'
+                  keep_time_key: false
+                  format: '/^(?<Pid>\d+) (?<Timestamp>.*) mysqld\_safe (?<Payload>.*)$/'
+        filter:
+          wsrep_severity:
+            type: record_transformer
+            tag: mysql.log
+            enable_ruby: true
+            remove_keys: priority
+            record:
+              - name: Severity
+                value: '${ {"Note"=>5,"Warning"=>4,"ERROR"=>3}.fetch(record["Severity"], 6) }'
+          mysql_record:
+            require:
+              - wsrep_severity
+            type: record_transformer
+            tag: mysql.log
+            enable_ruby: true
+            remove_keys: priority
+            record:
+              - name: service
+                value: mysql.service
+              - name: source
+                value: error_log
+              - name: severity_label
+                value: '${ {"TRACE"=>8,"DEBUG"=>7,"INFO"=>6,"NOTICE"=>5,"WARNING"=>4,"ERROR"=>3,"CRITICAL"=>2,"ALERT"=>1,"EMERGENCY"=>0}.key(record["Severity"].to_i) }'
+        match:
+          push_to_default:
+            tag: mysql.**
+            type: relabel
+            label: default_output
+{%- endif %}

metadata/service/master/cluster.yml

     galera_innodb_write_io_threads: 8
     galera_wsrep_slave_threads: 8
     galera_xtrabackup_parallel: 4
-    galera_error_log_enabled: false
+    galera_error_log_enabled: true
+    galera_error_log_path: /var/log/mysql/error.log
   galera:
     master:
       enabled: true
       wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
       xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
       error_log_enabled: ${_param:galera_error_log_enabled}
+      error_log_path: ${_param:galera_error_log_path}
   mysql:
     server:
       users:

metadata/service/master/container.yml

     galera_innodb_write_io_threads: 8
     galera_wsrep_slave_threads: 8
     galera_xtrabackup_parallel: 4
-    galera_error_log_enabled: false
+    galera_error_log_enabled: true
+    galera_error_log_path: /var/log/mysql/error.log
   kubernetes:
     control:
       configmap:
                 wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
                 xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
                 error_log_enabled: ${_param:galera_error_log_enabled}
+                error_log_path: ${_param:galera_error_log_path}
             mysql:
               server:
                 users:
                 - name: haproxy
                   host: localhost
                 - name: haproxy
-                  host: '%'
+                  host: '%'

metadata/service/master/single.yml

     galera_innodb_write_io_threads: 8
     galera_wsrep_slave_threads: 8
     galera_xtrabackup_parallel: 4
-    galera_error_log_enabled: false
+    galera_error_log_enabled: true
+    galera_error_log_path: /var/log/mysql/error.log
   galera:
     master:
       enabled: true
       wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
       xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
       error_log_enabled: ${_param:galera_error_log_enabled}
+      error_log_path: ${_param:galera_error_log_path}
   mysql:
     server:
       users:

metadata/service/slave/cluster.yml

     galera_innodb_write_io_threads: 8
     galera_wsrep_slave_threads: 8
     galera_xtrabackup_parallel: 4
-    galera_error_log_enabled: false
+    galera_error_log_enabled: true
+    galera_error_log_path: /var/log/mysql/error.log
   galera:
     slave:
       enabled: true
       wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
       xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
       error_log_enabled: ${_param:galera_error_log_enabled}
+      error_log_path: ${_param:galera_error_log_path}
   mysql:
     server:
       users:

metadata/service/support.yml

         enabled: true
       heka:
         enabled: true
+      fluentd:
+        enabled: true
       sensu:
         enabled: false
       sphinx: