Fix pillar tests

* Update run_tests.sh to the latest revision
  * drop odd .kitchen.vagrant.yml

Related-Bug: PROD-20730 (PROD:20730)

Change-Id: I367800a60ad17020700a76670d1216dfdfcfe692

.kitchen.vagrant.yml

@@ -1,36 +0,0 @@
----
-driver:
-  name: vagrant
-  vm_hostname: linux.ci.local
-  use_sudo: false
-  customize:
-    memory: 1024
-
-
-provisioner:
-  name: salt_solo
-  salt_install: bootstrap
-  salt_bootstrap_url: https://bootstrap.saltstack.com
-  salt_version: latest
-  require_chef: false
-  log_level: error
-  formula: linux
-  grains:
-    noservices: true
-  state_top:
-    base:
-      "*":
-        - linux
-  pillars:
-    top.sls:
-      base:
-        "*":
-          - linux
-
-platforms:
-- name: ubuntu-16.04
-- name: ubuntu-14.04
-- name: centos-7.3
-- name: centos-6.8
-
-# vim: ft=yaml sw=2 ts=2 sts=2 tw=125

linux/meta/fluentd.yml

@@ -1,3 +1,5 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
 {%- if grains.get('init') == 'systemd' %}
 agent:
   plugin:
@@ -54,7 +56,7 @@ agent:
             type: systemd
             tag: systemd.source
             path: /run/log/journal
-            pos_file: {{ pillar.fluentd.agent.dir.positiondb }}/systemd.source.pos
+            pos_file: {{ positiondb }}/systemd.source.pos
             entry:
               field_map:
                 MESSAGE: 'Payload'
@@ -142,7 +144,7 @@ agent:
             type: tail
             tag: linux.syslog
             path: /var/log/syslog
-            pos_file: {{ pillar.fluentd.agent.dir.positiondb }}/linux_syslog.pos
+            pos_file: {{ positiondb }}/linux_syslog.pos
             suppress_parse_error_log: true
             parser:
               type: regexp
@@ -170,3 +172,4 @@ agent:
             type: relabel
             label: default_metric
 {%- endif %}
+{%- endif %}

linux/network/hostname.sls

@@ -25,7 +25,6 @@ linux_enforce_hostname:
     - apply_hostname: True
     - retain_settings: True
 #}
-
 linux_enforce_hostname:
   cmd.run:
   - name: hostname {{ network.hostname }}

tests/pillar/network_openvswitch.sls

@@ -2,6 +2,7 @@ linux:
   system:
     enabled: true
     domain: local
+    name: linux
   network:
     enabled: true
     hostname: test01

tests/pillar/network_openvswitch_dpdk.sls

@@ -2,6 +2,7 @@ linux:
   system:
     enabled: true
     domain: local
+    name: linux
   network:
     enabled: true
     hostname: test01

tests/pillar/storage.sls

@@ -1,4 +1,12 @@
 linux:
+  system:
+    enabled: true
+    name: linux
+    domain: local
+  network:
+    enabled: true
+    hostname: linux
+    fqdn: linux.ci.local
   storage:
     enabled: true
     swap:

tests/pillar/system.sls

@@ -147,9 +147,10 @@ linux:
           enabled: true
           https: https://127.0.5.1:443
           #http: http://127.0.5.2:8080
-      apt-mk-salt:
-        source: "deb http://apt-mk.mirantis.com/xenial stable salt"
-        #key_url: http://apt-mk.mirantis.com/public.gpg
+      apt-salt:
+        source: "deb http://apt.mirantis.com/xenial stable salt"
+        #key_url: http://apt.mirantis.com/public.gpg
+        # pub   4096R/A76882D3 2015-06-17
         key: |
           -----BEGIN PGP PUBLIC KEY BLOCK-----
           Version: GnuPG v1
@@ -206,15 +207,15 @@ linux:
         architectures: amd64
         proxy:
           enabled: true
-      apt-mk-salt-nightly:
-        source: "deb http://apt-mk.mirantis.com/xenial nightly salt"
-        key_url: http://apt-mk.mirantis.com/public.gpg
+      apt-salt-nightly:
+        source: "deb http://apt.mirantis.com/xenial nightly salt"
+        key_url: http://apt.mirantis.com/public.gpg
         architectures: amd64
         proxy:
           enabled: false
-      apt-mk-extra-nightly:
-        source: "deb http://apt-mk.mirantis.com/xenial nightly extra"
-        key_url: http://apt-mk.mirantis.com/public.gpg
+      apt-extra-nightly:
+        source: "deb http://apt.mirantis.com/xenial nightly extra"
+        key_url: http://apt.mirantis.com/public.gpg
         architectures: amd64
     locale:
       en_US:

tests/pillar/system_banner.sls

@@ -1,6 +1,11 @@
 linux:
+  network:
+    enabled: true
+    hostname: linux
+    fqdn: linux.ci.local
   system:
     enabled: true
+    name: linux
     banner:
       enabled: true
       contents: |

tests/pillar/system_extra.sls

@@ -1,5 +1,9 @@
 
 linux:
+  network:
+    enabled: true
+    hostname: linux
+    fqdn: linux.ci.local
   system:
     auth:
       enabled: true

tests/pillar/system_file.sls

@@ -1,5 +1,10 @@
 linux:
+  network:
+    enabled: true
+    hostname: linux
+    fqdn: linux.ci.local
   system:
+    name: linux
     enabled: true
     file:
       /tmp/sample.txt:

tests/run_tests.sh

@@ -1,24 +1,39 @@
 #!/usr/bin/env bash
 
+###
+# Script requirments:
+#apt-get install -y python-yaml virtualenv git
+
+__ScriptVersion="2018.08.16"
+__ScriptName="run_tests.sh"
+__ScriptFullName="$0"
+__ScriptArgs="$*"
+
 set -e
 [ -n "$DEBUG" ] && set -x
 
 CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 METADATA=${CURDIR}/../metadata.yml
 FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
 
 ## Overrideable parameters
 PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
 BUILDDIR=${BUILDDIR:-${CURDIR}/build}
 VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
 DEPSDIR=${BUILDDIR}/deps
+SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
 
 SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
 SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
 SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
 SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
+
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
 
-SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}"
+IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
 
 if [ "x${SALT_VERSION}" != "x" ]; then
     PIP_SALT_VERSION="==${SALT_VERSION}"
@@ -26,24 +41,38 @@ fi
 
 ## Functions
 log_info() {
-    echo "[INFO] $*"
+    echo -e "[INFO] $*"
 }
 
 log_err() {
-    echo "[ERROR] $*" >&2
+    echo -e "[ERROR] $*" >&2
 }
 
 setup_virtualenv() {
     log_info "Setting up Python virtualenv"
+    dependency_check virtualenv
     virtualenv $VENV_DIR
     source ${VENV_DIR}/bin/activate
     python -m pip install salt${PIP_SALT_VERSION}
+    if [[ -f ${CURDIR}/test-requirements.txt ]]; then
+       python -m pip install -r ${CURDIR}/test-requirements.txt
+    fi
+}
+
+setup_mock_bin() {
+    # If some state requires a binary, a lightweight replacement for
+    # such binary can be put into MOCK_BIN_DIR for test purposes
+    if [ -d "${MOCK_BIN_DIR}" ]; then
+        PATH="${MOCK_BIN_DIR}:$PATH"
+        export PATH
+    fi
 }
 
 setup_pillar() {
     [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
     echo "base:" > ${SALT_PILLAR_DIR}/top.sls
     for pillar in ${PILLARDIR}/*; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
     done
@@ -53,9 +82,11 @@ setup_salt() {
     [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
     [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
     [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+    [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
 
     echo "base:" > ${SALT_FILE_DIR}/top.sls
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
     done
@@ -63,13 +94,14 @@ setup_salt() {
     cat << EOF > ${SALT_CONFIG_DIR}/minion
 file_client: local
 cachedir: ${SALT_CACHE_DIR}
+extension_modules:  ${SALT_CACHE_EXTMODS_DIR}
 verify_env: False
+minion_id_caching: False
 
 file_roots:
   base:
   - ${SALT_FILE_DIR}
   - ${CURDIR}/..
-  - /usr/share/salt-formulas/env
 
 pillar_roots:
   base:
@@ -79,13 +111,14 @@ EOF
 }
 
 fetch_dependency() {
+    # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
     dep_name="$(echo $1|cut -d : -f 1)"
     dep_source="$(echo $1|cut -d : -f 2-)"
     dep_root="${DEPSDIR}/$(basename $dep_source .git)"
     dep_metadata="${dep_root}/metadata.yml"
 
-    [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
-    [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
+    dependency_check git
+    [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
 
     log_info "Fetching dependency $dep_name"
     [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
@@ -95,6 +128,19 @@ fetch_dependency() {
     METADATA="${dep_metadata}" install_dependencies
 }
 
+link_modules(){
+    # Link modules *.py files to temporary salt-root
+    local SALT_ROOT=${1:-$SALT_FILE_DIR}
+    local SALT_ENV=${2:-$DEPSDIR}
+
+    mkdir -p "${SALT_ROOT}/_modules/"
+    # from git, development versions
+    find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
+      ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
+    done
+    salt_run saltutil.sync_all
+}
+
 install_dependencies() {
     grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
     (python - | while read dep; do fetch_dependency "$dep"; done) << EOF
@@ -115,19 +161,92 @@ salt_run() {
 }
 
 prepare() {
-    [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+    if [[ -f ${BUILDDIR}/.prepare_done ]]; then
+      log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
+      return
+    fi
+    [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
 
-    which salt-call || setup_virtualenv
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    setup_mock_bin
     setup_pillar
     setup_salt
     install_dependencies
+    link_modules
+    touch ${BUILDDIR}/.prepare_done
+}
+
+lint_releasenotes() {
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    source ${VENV_DIR}/bin/activate
+    reno lint ${CURDIR}/../
+}
+
+lint() {
+#    lint_releasenotes
+    log_err "TODO: lint_releasenotes"
 }
 
 run() {
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
+        salt_run grains.set 'noservices' False force=True
+
+        echo "Checking state ${FORMULA_NAME}.${state_name} ..."
         salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+        # Check that all files in 'meta' folder can be rendered using any valid pillar
+        for meta in `find ${FORMULA_META_DIR} -type f`; do
+            meta_name=$(basename ${meta})
+            echo "Checking meta ${meta_name} ..."
+            salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+              || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
+            cat ${SALT_CACHE_DIR}/${meta_name}
+        done
+    done
+}
+
+real_run() {
+    for pillar in ${PILLARDIR}/*.sls; do
+        state_name=$(basename ${pillar%.sls})
+        salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
+    done
+}
+
+run_model_validate(){
+  # Run modelschema.model_validate validation.
+  # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
+  # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
+  # Example:
+  # client.sls  client_auth.sls  server.sls  server_auth.sls
+  if [ -d ${SCHEMARDIR} ]; then
+    # model validator require py modules
+    fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
+    link_modules
+    salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
+    for role in ${SCHEMARDIR}/*.yaml; do
+      role_name=$(basename "${role%*.yaml}")
+      for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
+        pillar_name=$(basename "${pillar%*.sls}")
+        local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
+        log_info "model_validate ${_message}"
+        # Rendered Example:
+        # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
+        salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+      done
     done
+  else
+    log_info "${SCHEMARDIR} not found!";
+  fi
+}
+
+dependency_check() {
+  local DEPENDENCY_COMMANDS=$*
+
+  for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
+    which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
+  done
 }
 
 _atexit() {
@@ -143,6 +262,10 @@ _atexit() {
 }
 
 ## Main
+
+log_info "Running version: ${__ScriptVersion}"
+log_info "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
+
 trap _atexit INT TERM EXIT
 
 case $1 in
@@ -152,11 +275,23 @@ case $1 in
     prepare)
         prepare
         ;;
+    lint)
+        lint
+        ;;
     run)
         run
         ;;
+    real-run)
+        real_run
+        ;;
+    model-validate)
+       prepare
+       run_model_validate
+        ;;
     *)
         prepare
+#        lint
         run
+        run_model_validate
         ;;
 esac

tests/test-requirements.txt

@@ -0,0 +1,2 @@
+jsonschema
+reno