Merge branch 'master' into pr/27

README.rst

@@ -80,6 +80,7 @@ Additional mysql users:
                   - all privileges
 
 Additional check params:
+========================
 
 .. code-block:: yaml
 
@@ -92,6 +93,20 @@ Additional check params:
         - available_when_readonly: 1
         - port 9200
 
+InnoDB parameters
+=================
+
+- **innodb_buffer_pool_size** - the default value is 35% of the available ram
+
+Usage:
+.. code-block:: yaml
+
+    galera:
+        master:
+          innodb_buffer_pool_size: 1024M
+        slave:
+          innodb_buffer_pool_size: 1024M
+
 
 Usage
 =====
@@ -117,15 +132,15 @@ Galera monitoring command, performed from extra server
 
 .. code-block:: bash
 
-    Enter current password for root (enter for none): 
+    Enter current password for root (enter for none):
     OK, successfully used password, moving on...
 
     Setting the root password ensures that nobody can log into the MySQL
     root user without the proper authorisation.
 
     Set root password? [Y/n] y
-    New password: 
-    Re-enter new password: 
+    New password:
+    Re-enter new password:
     Password updated successfully!
     Reloading privilege tables..
      ... Success!
@@ -164,7 +179,7 @@ Galera monitoring command, performed from extra server
     Cleaning up...
 
 5. service mysql stop
-6. uncomment all wsrep* lines except first server, where leave only in my.cnf wsrep_cluster_address='gcomm://'; 
+6. uncomment all wsrep* lines except first server, where leave only in my.cnf wsrep_cluster_address='gcomm://';
 7. start first node
 8. Start third node which is connected to first one
 9. Start second node which is connected to third one

galera/files/my.cnf

@@ -38,7 +38,12 @@ query_cache_type=0
 
 innodb_file_format=Barracuda
 innodb_file_per_table=1
-innodb_buffer_pool_size=3138M
+{%- if service.innodb_buffer_pool_size is not defined %}
+  {% set buffer_pool_size = '%.0fM'%(grains['mem_total']*0.35) %}
+{%- else %}
+  {% set buffer_pool_size = service.innodb_buffer_pool_size %}
+{%- endif %}
+innodb_buffer_pool_size={{ buffer_pool_size }}
 innodb_log_file_size=627M
 innodb_read_io_threads=8
 innodb_write_io_threads=8
@@ -48,7 +53,6 @@ innodb_flush_method=O_DIRECT
 innodb_doublewrite=0
 innodb_autoinc_lock_mode=2
 innodb_locks_unsafe_for_binlog=1
-
 wsrep_cluster_address="gcomm://{% for member in service.members %}{{ member.host}}:4567{% if not loop.last %},{% endif %}{% endfor %}/?pc.wait_prim=no"
 wsrep_provider={{ service.wsrep_provider }}
 wsrep_cluster_name="openstack"

galera/master.sls

@@ -105,7 +105,7 @@ galera_conf_debian:
   - require:
     - pkg: galera_packages
 
-{%- endif %} 
+{%- endif %}
 
 galera_init_script:
   file.managed:
@@ -168,11 +168,26 @@ mysql_bootstrap_update_maint_password:
   - require:
     - cmd: galera_bootstrap_set_root_password
 
+galera_bootstrap_stop_service_pre:
+  cmd.run:
+  - name: mysqladmin -h localhost -u root -p{{ master.admin.password }} shutdown
+  {%- if not grains.get('noservices', False) %}
+  - ignore_retcode: true
+  - require:
+    - cmd: mysql_bootstrap_update_maint_password
+  {%- else %}
+  - onlyif: /bin/false
+  {%- endif %}
+
 galera_bootstrap_stop_service:
   service.dead:
   - name: {{ master.service }}
+  {%- if not grains.get('noservices', False) %}
   - require:
-    - cmd: mysql_bootstrap_update_maint_password
+    - cmd: galera_bootstrap_stop_service_pre
+  {%- else %}
+  - onlyif: /bin/false
+  {%- endif %}
 
 galera_bootstrap_init_config:
   file.managed:
@@ -180,7 +195,7 @@ galera_bootstrap_init_config:
   - source: salt://galera/files/my.cnf.init
   - mode: 644
   - template: jinja
-  - require: 
+  - require:
     - service: galera_bootstrap_stop_service
 
 galera_bootstrap_start_service_final:
@@ -209,7 +224,7 @@ galera_config:
   - source: salt://galera/files/my.cnf
   - mode: 644
   - template: jinja
-  - require_in: 
+  - require_in:
     - service: galera_service
 
 galera_service:

galera/meta/grafana.yml

@@ -8,6 +8,7 @@ dashboard:
     format: json
     template: galera/files/grafana_dashboards/mysql_influxdb.json
   main:
+    datasource: influxdb
     row:
       ost-middleware:
         title: Middleware

galera/meta/prometheus.yml

@@ -1,6 +1,15 @@
 {% raw %}
 server:
   alert:
+    GaleraServiceDown:
+      if: >-
+        mysql_up != 1
+      labels:
+        severity: warning
+        service: mysql
+      annotations:
+        summary: 'Galera service down'
+        description: 'Galera service is down on node {{ $labels.host }}'
     GaleraNodeNotReady:
       if: 'mysql_wsrep_ready != 1'
       for: 1m

galera/slave.sls

@@ -144,7 +144,7 @@ galera_init_start_service:
   {%- if grains.get('noservices') %}
   - onlyif: /bin/false
   {%- endif %}
-  - require: 
+  - require:
     - file: galera_run_dir
     - file: galera_init_script
   - timeout: 1800
@@ -167,14 +167,26 @@ mysql_bootstrap_update_maint_password:
   - require:
     - cmd: galera_bootstrap_set_root_password
 
+galera_bootstrap_stop_service_pre:
+  cmd.run:
+  - name: mysqladmin -h localhost -u root -p{{ slave.admin.password }} shutdown
+  {%- if not grains.get('noservices', False) %}
+  - ignore_retcode: true
+  - require:
+    - cmd: mysql_bootstrap_update_maint_password
+  {%- else %}
+  - onlyif: /bin/false
+  {%- endif %}
+
 galera_bootstrap_stop_service:
   service.dead:
   - name: {{ slave.service }}
-  {%- if grains.get('noservices') %}
+  {%- if not grains.get('noservices', False) %}
+  - require:
+    - cmd: galera_bootstrap_stop_service_pre
+  {%- else %}
   - onlyif: /bin/false
   {%- endif %}
-  - require:
-    - cmd: mysql_bootstrap_update_maint_password
 
 galera_bootstrap_init_config:
   file.managed:
@@ -182,7 +194,7 @@ galera_bootstrap_init_config:
   - source: salt://galera/files/my.cnf
   - mode: 644
   - template: jinja
-  - require: 
+  - require:
     - service: galera_bootstrap_stop_service
 
 galera_bootstrap_start_service_final:
@@ -191,7 +203,7 @@ galera_bootstrap_start_service_final:
   {%- if grains.get('noservices') %}
   - onlyif: /bin/false
   {%- endif %}
-  - require: 
+  - require:
     - file: galera_bootstrap_init_config
     - file: galera_bootstrap_script
   - timeout: 1800
@@ -212,7 +224,7 @@ galera_config:
   - source: salt://galera/files/my.cnf
   - mode: 644
   - template: jinja
-  - require_in: 
+  - require_in:
     - service: galera_service
 
 galera_service:

tests/run_tests.sh

@@ -6,11 +6,13 @@ set -e
 CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 METADATA=${CURDIR}/../metadata.yml
 FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
 
 ## Overrideable parameters
 PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
 BUILDDIR=${BUILDDIR:-${CURDIR}/build}
 VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
 DEPSDIR=${BUILDDIR}/deps
 
 SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
@@ -18,7 +20,7 @@ SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
 SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
 SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
 
-SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}"
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
 
 if [ "x${SALT_VERSION}" != "x" ]; then
     PIP_SALT_VERSION="==${SALT_VERSION}"
@@ -40,10 +42,20 @@ setup_virtualenv() {
     pip install salt${PIP_SALT_VERSION}
 }
 
+setup_mock_bin() {
+    # If some state requires a binary, a lightweight replacement for
+    # such binary can be put into MOCK_BIN_DIR for test purposes
+    if [ -d "${MOCK_BIN_DIR}" ]; then
+        PATH="${MOCK_BIN_DIR}:$PATH"
+        export PATH
+    fi
+}
+
 setup_pillar() {
     [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
     echo "base:" > ${SALT_PILLAR_DIR}/top.sls
     for pillar in ${PILLARDIR}/*; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
     done
@@ -56,6 +68,7 @@ setup_salt() {
 
     echo "base:" > ${SALT_FILE_DIR}/top.sls
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
     done
@@ -64,6 +77,7 @@ setup_salt() {
 file_client: local
 cachedir: ${SALT_CACHE_DIR}
 verify_env: False
+minion_id_caching: False
 
 file_roots:
   base:
@@ -118,6 +132,7 @@ prepare() {
     [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
 
     which salt-call || setup_virtualenv
+    setup_mock_bin
     setup_pillar
     setup_salt
     install_dependencies
@@ -125,9 +140,28 @@ prepare() {
 
 run() {
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         salt_run grains.set 'noservices' False force=True
+
+        echo "Checking state ${FORMULA_NAME}.${state_name} ..."
         salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+        # Check that all files in 'meta' folder can be rendered using any valid pillar
+        for meta in `find ${FORMULA_META_DIR} -type f`; do
+            meta_name=$(basename ${meta})
+            echo "Checking meta ${meta_name} ..."
+            salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+              || (log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1)
+            cat ${SALT_CACHE_DIR}/${meta_name}
+        done
+    done
+}
+
+real_run() {
+    for pillar in ${PILLARDIR}/*.sls; do
+        state_name=$(basename ${pillar%.sls})
+        salt_run --id=${state_name} state.sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
     done
 }
 
@@ -156,6 +190,9 @@ case $1 in
     run)
         run
         ;;
+    real-run)
+        real_run
+        ;;
     *)
         prepare
         run