Provisioning overhaul

CHANGELOG.rst

@@ -3,6 +3,13 @@
 salt-formula
 ============
 
+0.0.3
+-----
+
+- High availability
+- Cloud control updated
+- Virt control added
+
 0.0.2
 -----
 

README.rst

@@ -13,79 +13,15 @@ Sample pillars
 Salt master
 -----------
 
-Salt master with base environment and pillar metadata source
+Salt master with base production environment and pillar tree as metadata backend
 
-.. code-block:: yaml
-
-    salt:
-      master:
-        enabled: true
-        command_timeout: 5
-        worker_threads: 2
-        pillar:
-          engine: salt
-          source:
-            engine: git
-            address: 'git@repo.domain.com:salt/pillar-demo.git'
-            branch: 'master'
-        base_environment: prd
-        environment:
-          prd:
-            enabled: true
-            formula:
-              linux:
-                source: git
-                address: 'git@repo.domain.com:salt/formula-linux.git'
-                branch: 'master'
-              salt:
-                source: git
-                address: 'git@repo.domain.com:salt/formula-salt.git'
-                branch: 'master'
-              openssh:
-                source: git
-                address: 'git@repo.domain.com:salt/formula-openssh.git'
-                branch: 'master'
-
-Simple Salt master with base environment and custom states
-
-.. code-block:: yaml
-
-    salt:
-      master:
-        ...
-        environment:
-          base:
-            states:
-            - name: gitlab
-              source: git
-              address: 'git@repo.domain.cz:salt/state-gitlab.git'
-              branch: 'master'
-            formulas:
-            ...
-
-Salt master with reclass ENC
-
-.. code-block:: yaml
-
-    salt:
-      master:
-        enabled: true
-        ...
-        pillar:
-          engine: reclass
-          data_dir: /srv/salt/reclass
+.. literalinclude:: tests/pillar/master_single_pillar.sls
+   :language: yaml
 
-Salt master with windows repository
+Salt master with reclass ENC as metadata backend
 
-.. code-block:: yaml
-
-    salt:
-      master:
-        enabled: true
-        ...
-        windows_repo:
-          type: git
-          address: 'git@repo.domain.com:salt/win-packages.git'
+.. literalinclude:: tests/pillar/master_single_reclass.sls
+   :language: yaml
 
 Salt master with API
 
@@ -139,153 +75,70 @@ Salt master with custom handlers
     salt:
       master:
         enabled: true
-        command_timeout: 5
-        worker_threads: 2
-        environments:
-        - name: base
-          states:
-          - source: git
-            address: 'git@repo.domain.com:salt/state-ubuntu.git'
-            branch: 'master'
-          pillar:
-            source: git
-            address: 'git@repo.domain.com:salt/pillar-demo.git'
-            branch: 'master'
-        handlers:
-          name: logstash
-          type: udp
-          bind:
-            host: 127.0.0.1
-            port: 9999
+        handler:
+          handler01:
+            engine: udp
+            bind:
+              host: 127.0.0.1
+              port: 9999
       minion:
-        handlers:
-        - engine: udp
-          bind:
-            host: 127.0.0.1
-            port: 9999
-        - engine: zmq
-          bind:
-            host: 127.0.0.1
-            port: 9999
+        handler:
+          handler01:
+            engine: udp
+            bind:
+              host: 127.0.0.1
+              port: 9999
+          handler02:
+            engine: zmq
+            bind:
+              host: 127.0.0.1
+              port: 9999
+
 
 Salt minion
 -----------
 
-Simplest Salt minion
+Simplest Salt minion setup with central configuration node
 
 .. code-block:: yaml
 
-    salt:
-      minion:
-        enabled: true
-        master:
-          host: master.domain.com
-
-Multi-master Salt minion
+.. literalinclude:: tests/pillar/minion_master.sls
+   :language: yaml
 
-.. code-block:: yaml
+Multi-master Salt minion setup
 
-    salt:
-      minion:
-        enabled: true
-        masters:
-        -  host: master1.domain.com
-        -  host: master2.domain.com
+.. literalinclude:: tests/pillar/minion_multi_master.sls
+   :language: yaml
 
 Salt minion with salt mine options
 
-.. code-block:: yaml
-
-    salt:
-      minion:
-        enabled: true
-        master:
-          host: master.domain.com
-        mine:
-          interval: 60
-          module:
-            grains.items: []
-            network.interfaces: []
+.. literalinclude:: tests/pillar/minion_mine.sls
+   :language: yaml
 
 Salt minion with graphing dependencies
 
-.. code-block:: yaml
-
-    salt:
-      minion:
-        enabled: true
-        graph_states: true
-        master:
-          host: master.domain.com
+.. literalinclude:: tests/pillar/minion_graph.sls
+   :language: yaml
 
-Salt control (cloud/virt)
--------------------------
 
-Salt cloud with local OpenStack insecure (ignoring SSL cert errors) provider 
+Salt control (cloud/kvm/docker)
+-------------------------------
 
-.. code-block:: yaml
+Salt cloud with local OpenStack provider
 
-    salt:
-      control:
-        enabled: true
-        provider:
-          openstack_account:
-            engine: openstack
-            insecure: true
-            region: RegionOne
-            identity_url: 'https://10.0.0.2:35357'
-            tenant: devops
-            user: user
-            password: 'password'
-            fixed_networks:
-            - 123d3332-18be-4d1d-8d4d-5f5a54456554e
-            floating_networks:
-            - public
-            ignore_cidr: 192.168.0.0/16
+.. literalinclude:: tests/pillar/control_cloud_openstack.sls
+   :language: yaml
 
 Salt cloud with Digital Ocean provider
 
-.. code-block:: yaml
-
-    salt:
-      control:
-        enabled: true
-        provider:
-          dony1:
-            engine: digital_ocean
-            region: New York 1
-            client_key: xxxxxxx
-            api_key: xxxxxxx
+.. literalinclude:: tests/pillar/control_cloud_digitalocean.sls
+   :language: yaml
 
-Salt cloud with cluster definition
+Salt virt KVM cluster
 
-.. code-block:: yaml
+.. literalinclude:: tests/pillar/control_virt.sls
+   :language: yaml
 
-    salt:
-      control:
-        enabled: true
-        cluster:
-          devops_ase:
-            config:
-              engine: salt
-              host: 147.32.120.1
-            node:
-              proxy1.ase.cepsos.cz:
-                provider: cepsos_devops
-                image: Ubuntu12.04 x86_64
-                size: m1.medium
-              node1.ase.cepsos.cz:
-                provider: cepsos_devops
-                image: Ubuntu12.04 x86_64
-                size: m1.medium
-              node2.ase.cepsos.cz:
-                provider: cepsos_devops
-                image: Ubuntu12.04 x86_64
-                size: m1.medium
-              node3.ase.cepsos.cz:
-                provider: cepsos_devops
-                image: Ubuntu12.04 x86_64
-                size: m1.medium
 
 Usage
 =====
@@ -302,6 +155,7 @@ Debug LIBCLOUD for salt-cloud connection
 
     export LIBCLOUD_DEBUG=/dev/stderr; salt-cloud --list-sizes provider_name --log-level all
 
+
 Read more
 =========
 
@@ -313,6 +167,7 @@ Read more
 * https://github.com/saltstack-formulas/salt-formula
 * http://docs.saltstack.com/en/latest/topics/tutorials/multimaster.html
 
+
 salt-cloud
 ----------
 
@@ -323,3 +178,6 @@ salt-cloud
 * http://salt-cloud.readthedocs.org/en/latest/topics/rackspace.html
 * http://salt-cloud.readthedocs.org/en/latest/topics/map.html
 * http://docs.saltstack.com/en/latest/topics/tutorials/multimaster.html
+
+salt-virt
+---------

metadata/service/control/cloud_aws.yml

@@ -0,0 +1,5 @@
+parameters:
+  salt:
+    control:
+      enabled: true
+      cloud_enabled: true

metadata/service/control/cloud_do.yml

@@ -0,0 +1,5 @@
+parameters:
+  salt:
+    control:
+      enabled: true
+      cloud_enabled: true

metadata/service/control/cloud_os.yml

@@ -0,0 +1,5 @@
+parameters:
+  salt:
+    control:
+      enabled: true
+      cloud_enabled: true

metadata/service/control/maas.yml

@@ -0,0 +1,5 @@
+parameters:
+  salt:
+    control:
+      enabled: true
+      maas_enabled: true

metadata/service/control/virt.yml

@@ -0,0 +1,18 @@
+parameters:
+  salt:
+    control:
+      enabled: true
+      virt_enabled: true
+      size:
+        small:
+          cpu: 1
+          ram: 2
+          hdd: 10
+        medium:
+          cpu: 2
+          ram: 4
+          hdd: 20
+        large:
+          cpu: 4
+          ram: 8
+          hdd: 70

salt/control/cluster.sls → salt/control/cloud.sls

@@ -1,5 +1,23 @@
 {% from "salt/map.jinja" import control with context %}
-{%- if control.enabled %}
+{%- if control.enabled and control.cloud_enabled is defined %}
+
+salt_control_cloud_packages:
+  pkg.installed:
+    - names: {{ control.cloud_pkgs }}
+
+/etc/salt/control.providers:
+  file.managed:
+  - source: salt://salt/files/providers.conf
+  - user: root
+  - group: root
+  - template: jinja
+
+/etc/salt/control.profiles:
+  file.managed:
+  - source: salt://salt/files/profiles.conf
+  - user: root
+  - group: root
+  - template: jinja
 
 /srv/salt/cloud/maps:
   file.directory:
@@ -11,6 +29,8 @@
 
 {%- for cluster_name, cluster in control.cluster.iteritems() %}
 
+{%- if cluster.engine == "cloud" %}
+
 /srv/salt/cloud/maps/{{ cluster_name }}:
   file.managed:
   - source: salt://salt/files/map
@@ -42,6 +62,8 @@
 
 {%- endfor %}
 
+{%- endif %}
+
 {%- endfor %}
 
-{%- endif %}
+{%- endif %}

salt/control/docker.sls

@@ -0,0 +1,6 @@
+{% from "salt/map.jinja" import control with context %}
+{%- if control.enabled and control.docker_enabled is defined %}
+
+{# TODO: dockerng implementation #}
+
+{%- endif %}

salt/control/init.sls

@@ -1,4 +1,15 @@
+{%- if pillar.salt.control is defined %}
 include:
-- salt.control.service
-- salt.control.provider
-- salt.control.cluster
+{%- if pillar.salt.control.cloud_enabled is defined %}
+- salt.control.cloud
+{%- endif %}
+{%- if pillar.salt.control.docker_enabled is defined %}
+- salt.control.docker
+{%- endif %}
+{%- if pillar.salt.control.maas_enabled is defined %}
+- salt.control.maas
+{%- endif %}
+{%- if pillar.salt.control.virt_enabled is defined %}
+- salt.control.virt
+{%- endif %}
+{%- endif %}

salt/control/maas.sls

@@ -0,0 +1,22 @@
+{% from "salt/map.jinja" import control with context %}
+{%- if control.enabled and control.maas_enabled is defined %}
+
+salt_control_maas_packages:
+  pkg.installed:
+    - names: {{ control.maas_pkgs }}
+
+{%- for cluster_name, cluster in control.cluster.iteritems() %}
+
+{%- if cluster.engine == "maas" %}
+
+{%- for node_name, node in cluster.node.iteritems() %}
+
+{# TODO: mass.server_active implementation #}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endif %}

salt/control/provider.sls

@@ -1,18 +0,0 @@
-{% from "salt/map.jinja" import control with context %}
-{%- if control.enabled %}
-
-/etc/salt/cloud.providers:
-  file.managed:
-  - source: salt://salt/files/providers.conf
-  - user: root
-  - group: root
-  - template: jinja
-
-/etc/salt/cloud.profiles:
-  file.managed:
-  - source: salt://salt/files/profiles.conf
-  - user: root
-  - group: root
-  - template: jinja
-
-{%- endif %}

salt/control/service.sls

@@ -1,26 +0,0 @@
-{% from "salt/map.jinja" import control with context %}
-{%- if control.enabled %}
-
-{%- if control.pkgs is defined and control.pkgs|length > 0 %}
-
-salt_control_packages:
-  pkg.installed:
-    - names: {{ control.pkgs }}
-
-{%- else %}
-{# No system packages defined, install with pip #}
-
-salt_control_packages:
-  pkg.installed:
-  - name: python-pip
-
-{%- for package in control.python_pkgs %}
-{{ package }}:
-  pip.installed:
-  - require:
-    - pkg: salt_control_packages
-{%- endfor %}
-
-{%- endif %}
-
-{%- endif %}

salt/control/virt.sls

@@ -0,0 +1,39 @@
+{% from "salt/map.jinja" import control with context %}
+{%- if control.enabled and control.virt_enabled is defined %}
+
+salt_control_virt_packages:
+  pkg.installed:
+    - names: {{ control.virt_pkgs }}
+
+{%- for package in control.virt_pips %}
+
+{{ package }}:
+  pip.installed:
+  - require:
+    - pkg: salt_control_virt_packages
+
+{%- endfor %}
+
+{%- for cluster_name, cluster in control.cluster.iteritems() %}
+
+{%- if cluster.engine == "virt" %}
+
+{%- for node_name, node in cluster.node.iteritems() %}
+
+{%- set size = control.size.get(node.size) %}
+
+salt_control_virt_{{ cluster_name }}_{{ node_name }}:
+  module.run:
+  - name: virt.init
+  - m_name: {{ node_name }}_{{ cluster.domain }}
+  - cpu: {{ size.cpu }}
+  - mem: {{ size.ram }}
+  - image: salt://{{ node.image }}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endif %}

salt/files/minion.conf

@@ -2,12 +2,20 @@
 {%- from "linux/map.jinja" import system with context %}
 
 {%- if minion.masters is defined %}
+
 master:
 {%- for master in minion.masters %}
 - {{ master.host }}
 {%- endfor %}
+
+{%- if minion.master_type == "failover" %}
+master_type: failover
+master_shuffle: True
+master_alive_interval: 60
 {%- else %}
+
 master: {{ minion.master.host }}
+
 {%- endif %}
 
 id: {{ system.name }}.{{ system.domain }}

salt/map.jinja

@@ -95,6 +95,7 @@
     },
 }, merge=salt['pillar.get']('salt:minion')) %}
 
+
 {% set api = salt['grains.filter_by']({
     'Debian': {
         'pkgs': ['salt-api'],
@@ -106,15 +107,22 @@
     },
 }, merge=salt['pillar.get']('salt:api')) %}
 
+
 {% set control = salt['grains.filter_by']({
     'Debian': {
-        'python_pkgs': ['apache-libcloud', 'netaddr'],
-        'pkgs': ['python-netaddr', 'python-libcloud'],
+        'cloud_pips': ['apache-libcloud', 'netaddr'],
+        'cloud_pkgs': ['python-netaddr', 'python-libcloud'],
+        'virt_pips': ['libvirt-python',],
+        'virt_pkgs': ['libvirt-dev', 'pkg-config'],
+        'maas_pkgs': ['python-maas-client',],
         'cluster': {},
     },
     'RedHat': {
-        'python_pkgs': ['apache-libcloud', 'netaddr'],
-        'pkgs': ['python-netaddr', 'python-libcloud'],
+        'cloud_pips': ['apache-libcloud', 'netaddr'],
+        'cloud_pkgs': ['python-netaddr', 'python-libcloud'],
+        'virt_pips': ['libvirt-python',],
+        'virt_pkgs': ['libvirt-dev', 'pkg-config'],
+        'maas_pkgs': ['python-maas-client',],
         'cluster': {},
     },
 }, merge=salt['pillar.get']('salt:control')) %}

tests/pillar/control_cloud_digitalocean.sls

@@ -0,0 +1,25 @@
+salt:
+  control:
+    enabled: true
+    provider:
+      digitalocean_account:
+        engine: digital_ocean
+        region: New York 1
+        client_key: xxxxxxx
+        api_key: xxxxxxx
+    cluster:
+      dc01_prd:
+        domain: dc01.prd.domain.com
+        engine: cloud
+        config:
+          engine: salt
+          host: master.dc01.domain.com
+        node:
+          ubuntu1:
+            provider: digitalocean_account
+            image: Ubuntu14.04 x86_64
+            size: m1.medium
+          ubuntu2:
+            provider: digitalocean_account
+            image: Ubuntu14.04 x86_64
+            size: m1.medium

tests/pillar/control_cloud_openstack.sls

@@ -0,0 +1,33 @@
+salt:
+  control:
+    enabled: true
+    provider:
+      openstack_account:
+        engine: openstack
+        insecure: true
+        region: RegionOne
+        identity_url: 'https://10.0.0.2:35357'
+        tenant: project 
+        user: user
+        password: 'password'
+        fixed_networks:
+        - 123d3332-18be-4d1d-8d4d-5f5a54456554e
+        floating_networks:
+        - public
+        ignore_cidr: 192.168.0.0/16
+    cluster:
+      dc01_prd:
+        domain: dc01.prd.domain.com
+        engine: cloud
+        config:
+          engine: salt
+          host: master.dc01.domain.com
+        node:
+          ubuntu1:
+            provider: openstack_account
+            image: Ubuntu14.04 x86_64
+            size: m1.medium
+          ubuntu2:
+            provider: openstack_account
+            image: Ubuntu14.04 x86_64
+            size: m1.medium

tests/pillar/control_virt.sls

@@ -0,0 +1,32 @@
+salt:
+  control:
+    enabled: true
+    size:
+      small:
+        cpu: 1
+        ram: 1
+        hdd: 10
+      medium:
+        cpu: 2
+        ram: 4
+        hdd: 20
+      large:
+        cpu: 4
+        ram: 8
+        hdd: 70
+    cluster:
+      vpc20_infra:
+        domain: neco.virt.domain.com
+        engine: virt
+        config:
+          engine: salt
+          host: master.domain.com
+        node:
+          ubuntu1:
+            provider: node01.domain.com
+            image: ubuntu.qcow
+            size: medium
+          ubuntu2:
+            provider: node02.domain.com
+            image: bubuntu.qcomw
+            size: small

tests/pillar/master_single.sls → tests/pillar/master_single_pillar.sls

@@ -10,16 +10,19 @@ salt:
     command_timeout: 5
     worker_threads: 2
     base_environment: prd
+    environment:
+      prd:
+        formula:
+          service01:
+            source: git
+            address: 'git@git.domain.com/service01-formula.git'
+            revision: master
+          service02:
+            source: pkg
+            name: salt-formula-service02 
     pillar:
       engine: salt
       source:
         engine: git
         address: 'git@repo.domain.com:salt/pillar-demo.git'
         branch: 'master'
-    environment:
-      prd:
-        formula:
-          memcached:
-            source: git
-            address: 'git@git.domain.com/memcached-formula.git'
-            revision: master

tests/pillar/master_single_reclass.sls

@@ -17,13 +17,16 @@ salt:
     command_timeout: 5
     worker_threads: 2
     base_environment: prd
-    pillar:
-      engine: reclass
-      data_dir: /srv/salt/reclass
     environment:
       prd:
         formula:
-          memcached:
+          service01:
             source: git
-            address: 'git@git.domain.com/memcached-formula.git'
+            address: 'git@git.domain.com/service01-formula.git'
             revision: master
+          service02:
+            source: pkg
+            name: salt-formula-service02
+    pillar:
+      engine: reclass
+      data_dir: /srv/salt/reclass

tests/pillar/minion_graph.sls

@@ -0,0 +1,4 @@
+salt:
+  minion:
+    enabled: true
+    graph_states: true

tests/pillar/minion_local.sls → tests/pillar/minion_local_pillar.sls

@@ -7,4 +7,4 @@ salt:
       source:
         engine: git
         address: 'git@repo.domain.com:salt/pillar-demo.git'
-        branch: 'master'
+        branch: 'master'

tests/pillar/minion_master.sls

@@ -2,9 +2,4 @@ salt:
   minion:
     enabled: true
     master:
-      host: 127.0.0.1
-    mine:
-      interval: 60
-      module:
-        grains.items: []
-        network.interfaces: []
+      host: config01.dc01.domain.com

tests/pillar/minion_mine.sls

@@ -0,0 +1,8 @@
+salt:
+  minion:
+    enabled: true
+    mine:
+      interval: 60
+      module:
+        grains.items: []
+        network.interfaces: []

tests/pillar/minion_multi_master.sls

@@ -0,0 +1,6 @@
+salt:
+  minion:
+    enabled: true
+    masters:
+    - host: config01.dc01.domain.com
+    - host: config02.dc01.domain.com

tests/pillar/minion_multi_master_failover.sls

@@ -0,0 +1,7 @@
+salt:
+  minion:
+    enabled: true
+    masters:
+    - host: config01.dc01.domain.com
+    - host: config02.dc01.domain.com
+    master_type: failover