implement test harness

.gitignore

@@ -1,2 +1,12 @@
+*.egg
+*.egg-info/
+*.py[cod]
 .*.sw?
+.env
+.pytest_cache/
+.venv/
 /.idea/
+__pycache__/
+Dockerfile.*_*
+ignore/
+tmp/

.travis.yml

@@ -0,0 +1,24 @@
+env:
+  matrix:
+    - OS_ID: centos_master_2017.7.2
+    - OS_ID: debian_master_2017.7.2
+    - OS_ID: opensuse_master_2017.7.2
+    - OS_ID: ubuntu_master_2016.11.3
+    - OS_ID: ubuntu_master_2017.7.2
+
+sudo: required
+
+language: python
+
+services:
+- docker
+
+before_install:
+- pip install Jinja2
+- python ${TRAVIS_BUILD_DIR}/tools/filltmpl.py nginx ${OS_ID}
+
+install:
+- docker build --force-rm -t "nginx:salt-testing-${OS_ID}" -f "Dockerfile.${OS_ID}" .
+
+script:
+- ./tools/run-tests.sh nginx ${OS_ID}

Makefile

@@ -0,0 +1,80 @@
+FORMULA_NAME = "nginx"
+PWD = $(shell pwd)
+
+# ---------------------------------------------------------------
+define render_dockerfile
+	python $(PWD)/tools/filltmpl.py $(FORMULA_NAME) $(1)
+endef
+
+define docker_build
+	docker build --force-rm -t $(FORMULA_NAME):salt-testing-$(1) -f Dockerfile.$(1) .
+endef
+
+define docker_run_local
+	docker run --rm -v $(PWD):/opt/$(FORMULA_NAME)-formula --env=STAGE=TEST -h salt-testing-$(1) --name salt-testing-$(1) -it $(FORMULA_NAME):salt-testing-$(1) /bin/bash
+endef
+
+define run_tests
+	./tools/run-tests.sh $(FORMULA_NAME) $(1)
+endef
+
+# --- convenience functions -------------------------------------
+define build_thing
+	$(call render_dockerfile,$(1)) && $(call docker_build,$(1))
+endef
+
+define run_local_tests
+	$(call build_thing,$(1)) && $(call run_tests,$(1))
+endef
+
+define run_local
+	$(call build_thing,$(1)) && $(call docker_run_local,$(1))
+endef
+
+# ---------------------------------------------------------------
+setup:
+	pip install Jinja2
+
+clean:
+	find . -name '*.pyc' -exec rm '{}' ';'
+	rm -rf Dockerfile.*
+	# delete pytest caches...
+	# rm -rf tests/pytests/*/.pytest_cache
+	# rm -rf tests/pytests/*/__pycache__
+	rm -rf tests/pytests/apply-all-tests/.pytest_cache
+	rm -rf tests/pytests/apply-all-tests/__pycache__
+
+# --- centos_master_2017.7.2 ------------------------------------
+test-centos_master_2017.7.2: clean
+	$(call run_local_tests,centos_master_2017.7.2)
+
+local-centos_master_2017.7.2: clean
+	$(call run_local,centos_master_2017.7.2)
+
+# --- debian_master_2017.7.2 ------------------------------------
+test-debian_master_2017.7.2: clean
+	$(call run_local_tests,debian_master_2017.7.2)
+
+local-debian_master_2017.7.2: clean
+	$(call run_local,debian_master_2017.7.2)
+
+# --- opensuse_master_2017.7.2 ------------------------------------
+test-opensuse_master_2017.7.2: clean
+	$(call run_local_tests,opensuse_master_2017.7.2)
+
+local-opensuse_master_2017.7.2: clean
+	$(call run_local,opensuse_master_2017.7.2)
+
+# --- ubuntu_master_2016.11.3 ------------------------------------
+test-ubuntu_master_2016.11.3: clean
+	$(call run_local_tests,ubuntu_master_2016.11.3)
+
+local-ubuntu_master_2016.11.3: clean
+	$(call run_local,ubuntu_master_2016.11.3)
+
+# --- ubuntu_master_2017.7.2 ------------------------------------
+test-ubuntu_master_2017.7.2: clean
+	$(call run_local_tests,ubuntu_master_2017.7.2)
+
+local-ubuntu_master_2017.7.2: clean
+	$(call run_local,ubuntu_master_2017.7.2)

README.rst

@@ -112,3 +112,48 @@ the upstream phusion passenger repository with `install_from_phusionpassenger: t
 Nginx will also be installed from that repository, as it needs to be modified to
 allow the passenger module to work.
 
+
+
+Running Tests
+=============
+
+This test runner was implemented using the formula-test-harness_ project.
+
+Tests will be run on the following base images:
+
+* ``simplyadrian/allsalt:centos_master_2017.7.2``
+* ``simplyadrian/allsalt:debian_master_2017.7.2``
+* ``simplyadrian/allsalt:opensuse_master_2017.7.2``
+* ``simplyadrian/allsalt:ubuntu_master_2016.11.3``
+* ``simplyadrian/allsalt:ubuntu_master_2017.7.2``
+
+Local Setup
+-----------
+
+.. code-block:: shell
+
+   pip install -U virtualenv
+   virtualenv .venv
+   source .venv/bin/activate
+   make setup
+
+Run tests
+---------
+
+* ``make test-centos_master_2017.7.2``
+* ``make test-debian_master_2017.7.2``
+* ``make test-opensuse_master_2017.7.2``
+* ``make test-ubuntu_master_2016.11.3``
+* ``make test-ubuntu_master_2017.7.2``
+
+Run Containers
+--------------
+
+* ``make local-centos_master_2017.7.2``
+* ``make local-debian_master_2017.7.2``
+* ``make local-opensuse_master_2017.7.2``
+* ``make local-ubuntu_master_2016.11.3``
+* ``make local-ubuntu_master_2017.7.2``
+
+
+.. _formula-test-harness: https://github.com/intuitivetechnologygroup/formula-test-harness

tests/pytests/apply-all-tests/test_000_apply_state.py

@@ -0,0 +1,23 @@
+from subprocess import check_output
+from unittest import TestCase
+
+
+class ApplyStateTest(TestCase):
+
+    def test_000_apply(self):
+        state_apply_response = check_output(["salt-call", "--local", "state.apply"])
+        print('')
+        print('-' * 50)
+        print('state_apply_response:')
+        print(state_apply_response)
+        print('-' * 50)
+        print('')
+
+        state_apply_response = state_apply_response.split('\n')
+        summary = state_apply_response[-8:]
+        failed = 0
+        for line in summary:
+            if line.startswith('Failed:'):
+                failed = int(line.split(':').pop().strip())
+
+        self.assertEqual(failed, 0)

tests/srv/salt/top.sls

@@ -0,0 +1,3 @@
+base:
+  '*':
+    - nginx

tools/filltmpl.py

@@ -0,0 +1,27 @@
+import os
+import sys
+
+from jinja2 import Template
+
+# base/tests
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+# base
+base_path = os.path.dirname(dir_path)
+
+
+if __name__ == '__main__':
+    formula_name = sys.argv[1]
+    image_tag = sys.argv[2]
+
+    template = Template(
+        open(os.path.join(dir_path, 'templates', 'Dockerfile.j2')).read()
+    )
+
+    dockerfile = template.render({
+        'formula_name': formula_name,
+        'image_tag': image_tag
+    })
+
+    with open(os.path.join(base_path, 'Dockerfile.{}'.format(image_tag)), 'w') as fh:
+        fh.write(dockerfile)

tools/run-tests.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ev
+
+test -z $2 && echo "Usage: ${0} FORMULA_NAME OS_ID" && exit 1
+export FORMULA_NAME=$1
+export OS_ID=$2
+
+
+function docker-run-pytest() {
+    docker run --rm \
+        -v "$@":/opt/tests \
+        --env=STAGE=TEST \
+        -h "salt-testing-${OS_ID}" \
+        --name "salt-testing-${OS_ID}" \
+        -it ${FORMULA_NAME}:"salt-testing-${OS_ID}" \
+        pytest -sv /opt/tests
+}
+
+for i in $(find $PWD/tests/pytests/* -maxdepth 0 -type d); do
+    docker-run-pytest $i;
+done

tools/templates/Dockerfile.j2

@@ -0,0 +1,14 @@
+FROM simplyadrian/allsalt:{{ image_tag }}
+
+{% if 'debian' in image_tag or 'ubuntu' in image_tag -%}
+RUN apt-get update && \
+    apt-get install -y python-pip
+{% endif %}
+
+RUN pip install pytest && \
+    sed -i "s/#master: salt/master: localhost/g" /etc/salt/minion
+
+ADD tests/srv /srv
+ADD {{ formula_name }} /srv/salt/{{ formula_name }}
+
+WORKDIR /srv/salt