allow-multiple-ext-pillars-and-reclass-options

Change-Id: I336b0825712218e2381e1654251765a24581da00

.kitchen.yml

         master_ssh_root.sls: tests/pillar/master_ssh_minion_root.sls
         master_formulas.sls: tests/pillar/master_formulas.sls
 
+  - name: master-extpillar-composite
+    provisioner:
+      pillars-from-files:
+        salt.sls: tests/pillar/master_single_extpillars.sls
+
+  - name: master-extpillar-reclass
+    provisioner:
+      pillars-from-files:
+        salt.sls: tests/pillar/master_single_extreclass.sls
+
   - name: control-default
     provisioner:
       grains:

README.rst

 .. literalinclude:: tests/pillar/master_single_reclass.sls
    :language: yaml
 
+Salt master with multiple ext_pillars
+
+.. literalinclude:: tests/pillar/master_single_extpillars.sls
+   :language: yaml
+
 Salt master with API
 
 .. literalinclude:: tests/pillar/master_api.sls
       master:
         state_output: changes
 
+
 Salt synchronise node pillar and modules after start
 
 .. code-block:: yaml
     salt-call event.send 'reclass/minion/classify' "{'node_master_ip': '$config_host', 'node_ip': '${node_ip}', 'node_domain': '$node_domain', 'node_cluster': '$node_cluster', 'node_hostname': '$node_hostname', 'node_os': '$node_os'}"
 
 
+
+Encrypted pillars
+-----------------
+
+Note: NACL + below configuration will be available in Salt > 2017.7.
+
+External resources:
+
+- Tutorial to configure salt + reclass ext_pillar and nacl: http://apealive.net/post/2017-09-salt-nacl-ext-pillar/
+- Saltstack documentation: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.nacl.html
+
+Configure salt NACL module:
+
+.. code-block:: shell
+
+  pip install --upgrade libnacl===1.5.2
+  salt-call --local nacl.keygen /etc/salt/pki/master/nacl
+
+    local:
+        saved sk_file:/etc/salt/pki/master/nacl  pk_file: /etc/salt/pki/master/nacl.pub
+
+
+.. code-block:: yaml
+
+    salt:
+      master:
+        pillar:
+          reclass: *reclass
+          nacl:
+            index: 99
+        nacl:
+          box_type: sealedbox
+          sk_file: /etc/salt/pki/master/nacl
+          pk_file: /etc/salt/pki/master/nacl.pub
+          #sk: None
+          #pk: None
+
+NACL encrypt secrets:
+
+  salt-call --local nacl.enc 'my_secret_value' pk_file=/etc/salt/pki/master/nacl.pub
+    hXTkJpC1hcKMS7yZVGESutWrkvzusXfETXkacSklIxYjfWDlMJmR37MlmthdIgjXpg4f2AlBKb8tc9Woma7q
+  # or
+  salt-run nacl.enc 'myotherpass'
+    ADDFD0Rav6p6+63sojl7Htfrncp5rrDVyeE4BSPO7ipq8fZuLDIVAzQLf4PCbDqi+Fau5KD3/J/E+Pw=
+
+
+NACL encrypted values on pillar:
+
+Use Boxed syntax `NACL[CryptedValue=]` to encode value on pillar:
+
+.. code-block:: yaml
+
+  my_pillar:
+    my_nacl:
+        key0: unencrypted_value
+        key1: NACL[hXTkJpC1hcKMS7yZVGESutWrkvzusXfETXkacSklIxYjfWDlMJmR37MlmthdIgjXpg4f2AlBKb8tc9Woma7q]
+
+NACL large files:
+
+.. code-block:: shell
+  salt-call nacl.enc_file /tmp/cert.crt out=/srv/salt/env/dev/cert.nacl
+  # or more advanced
+  cert=$(cat /tmp/cert.crt)
+  salt-call --out=newline_values_only nacl.enc_pub data="$cert" > /srv/salt/env/dev/cert.nacl
+
+
+NACL within template/native pillars:
+
+  pillarexample:
+      user: root
+      password1: {{salt.nacl.dec('DRB7Q6/X5gGSRCTpZyxS6hlbWj0llUA+uaVyvou3vJ4=')|json}}
+      cert_key: {{salt.nacl.dec_file('/srv/salt/env/dev/certs/example.com/cert.nacl')|json}}
+      cert_key2: {{salt.nacl.dec_file('salt:///certs/example.com/cert2.nacl')|json}}
+
 Salt syndic
 -----------
 

metadata/service/master/nacl/init.yml

+salt:
+  master:
+    nacl:
+      box_type: sealedbox
+      sk_file: /etc/salt/pki/master/nacl
+      pk_file: /etc/salt/pki/master/nacl.pub

metadata/service/master/pillar/composite/gpg.yml

+classes:
+- service.master.pillar.composite
+parameters:
+  salt:
+    master:
+      pillar:
+        gpg:
+          index: 99
+

metadata/service/master/pillar/composite/init.yml

+parameters:
+  salt:
+    master:
+      pillar:
+        engine: composite

metadata/service/master/pillar/composite/nacl.yml

+classes:
+- service.master.nacl
+- service.master.pillar.composite
+parameters:
+  salt:
+    master:
+      pillar:
+        nacl:
+          # if order is provided 99 is used to compose "99-nacl" key name which is later used to order entries
+          index: 99
+

metadata/service/master/pillar/composite/reclass.yml

+classes:
+- service.master.pillar.composite
+parameters:
+  salt:
+    master:
+      pillar:
+        reclass:
+          index: 1
+          storage_type: yaml_fs
+          inventory_base_uri: /srv/salt/reclass
+          #class_mappings: []
+          propagate_pillar_data_to_reclass: False
+          ignore_class_notfound: False
+          #ignore_class_regexp: []

metadata/service/master/pillar/composite/saltclass.yml

+classes:
+- service.master.pillar.composite
+parameters:
+  salt:
+    master:
+      pillar:
+        saltclass:
+          path: /srv/salt/saltclass

metadata/service/master/pillar/reclass.yml

+parameters:
+  salt:
+    master:
+      pillar:
+        engine: reclass
+        reclass:
+          storage_type: yaml_fs
+          inventory_base_uri: /srv/salt/reclass

metadata/service/master/pillar/salt.yml

+parameters:
+  salt:
+    master:
+      pillar:
+        engine: salt
+        salt:
+          path: /srv/salt/pillar

metadata/service/master/pillar/saltclass.yml

+parameters:
+  salt:
+    master:
+      pillar:
+        engine: saltclass
+        saltclass:
+          path: /srv/salt/saltclass

metadata/service/minion/local.yml

       local: true
       pillar:
         engine: reclass
-        data_dir: /srv/salt/reclass

releasenotes/config.yaml

+---
+# Usage:
+#
+# reno list
+# reno new slug-title --edit
+# reno report --no-show-source
+
+# Change prelude_section_name to 'summary' from default value prelude
+prelude_section_name: summary
+show_source: False
+sections:
+  # summary/prelude section is always included
+  - [features, New Features]
+  - [issues, Known Issues]
+  - [fixes, Bug Fixes]
+  - [other, Other Notes]
+template: |
+  ---
+  # Author the following sections or remove the section if it is not related.
+  # Use one release note per a feature.
+  #
+  # If you miss a section from the list below, please first submit a review
+  # adding it to releasenotes/config.yaml.
+  #
+  summary: >
+    This section is not mandatory. Use it to highlight the change.
+
+  features:
+    - Use list to record summary of features.
+    - |
+      Provide detailed description with examples.
+      Format with reStructuredText.
+
+      .. code-block:: text
+
+         provide model/formula pillar snippets
+
+  issues:
+    - Use list to record known limitations.
+
+  fixes:
+    - Use list to record summary of fixes.
+      Quick and dirty `git log --oneline`.
+
+  other:
+    - Author additional notes for the release.
+    - Format with reStructuredText.
+    - |
+        Use this section if note is not related to one of the common sections:
+        features, issues, upgrade, deprecations, security, fixes, api, cli
+
+        * list item 1
+        * list item 2
+
+        .. code-block:: yaml
+
+          formula:
+            example:
+              enabled: true

releasenotes/notes/add-releasenotes-9c076c7ee8fbe2a4.yaml

+---
+summary: >
+  Use "reno", an releasenotes configuration tool to record release notes.
+  Documentation: https://docs.openstack.org/reno/latest
+
+  To list/create/show release notes, run following commands:
+
+    .. code-block:: shell
+
+      reno list
+      reno new releasenote-slug-title --edit
+      # use favored $EDITOR to update the note
+      # git add/commit releasenotes/* as usual
+      reno report --no-show-source
+
+other:
+  - |
+    Added `reno <https://docs.openstack.org/reno/latest>_` configuration to the repository.

releasenotes/notes/allow-multiple-ext-pillars-885d28dc8a18ab99.yaml

+---
+features:
+  - |
+    Added option to define multiple ext_pillars.
+    Example usage:
+
+    .. code-block:: text
+
+      salt:
+        master:
+          pillar:
+            engine: composite
+            reclass:
+              storage_type: yaml_fs
+              inventory_base_uri: /srv/salt/reclass_encrypted
+            nacl:
+              index: 99
+        nacl:
+          sk_file: /etc/salt/pki/master/nacl
+          pk_file: /etc/salt/pki/master/nacl.pub

salt/files/master.conf

 {%- endif %}
 
 {%- if master.pillar.engine == 'salt' %}
-
 pillar_roots:
   base:
-  - /srv/salt/pillar
-
+  - {{ master.pillar.get('salt', {}).get('path', '/srv/salt/pillar') }}
 {%- endif %}
 
-{%- if master.pillar.engine == 'reclass' %}
+{%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined)  %}
 
 reclass: &reclass
-  storage_type: yaml_fs
-  inventory_base_uri: /srv/salt/reclass
+  storage_type: {{ master.pillar.get('reclass', {'storage_type': 'yaml_fs'}).storage_type }}
+  inventory_base_uri: {{ master.pillar.get('reclass', {'inventory_base_uri': '/srv/salt/reclass'}).inventory_base_uri }}
+  {%- if master.pillar.reclass.reclass_source_path is defined %}
+  reclass_source_path: {{ master.pillar.reclass.reclass_source_path }}
+  {%- endif %}
+  {%- if master.pillar.reclass.get('class_mappings', [])|length > 0 %}
+  class_mappings:
+  {%- for mapping in master.pillar.reclass.class_mappings %}
+  - {{ mapping.target }} {{ mapping.class }}
+  {%- endfor %}
+  {%- endif %}
+  {%- if master.pillar.reclass.get('propagate_pillar_data_to_reclass', False) == True %}
+  propagate_pillar_data_to_reclass: {{ master.pillar.reclass.propagate_pillar_data_to_reclass }}
+  {%- endif %}
+  {%- if master.pillar.reclass.get('ignore_class_notfound', False) == True %}
+  # Below option is not available in upstream reclass, and require fork https://github.com/salt-formulas/reclass
+  ignore_class_notfound: {{ master.pillar.reclass.ignore_class_notfound }}
+  ignore_class_regexp: {{ master.pillar.reclass.ignore_class_regexp }}
+  {%- endif %}
+{%- endif %}
+
+{%- if master.pillar.engine == 'saltclass' or (master.pillar.engine == 'composite' and master.pillar.saltclass is defined )  %}
+
+saltclass: &saltclass
+  path: {{ master.pillar.saltclass.get('path', '/srv/salt/saltclass') }}
+{%- endif %}
+
+
+{%- if master.pillar.engine in ['composite', 'reclass', 'saltclass'] %}
+{# generate indexed list of ext_engines #}
+{# NONE: Might be rewritten, once proved to work properly, with filters: #}
+{# NONE: select('mapping')|selectattr('_index')|sort(attribute='_index') #}
+{%- set ext_engines = {} %}
+{%- for name,engine in master.pillar.iteritems() %}
+{%-   if not engine is mapping %}{% continue %}{% endif %}
+{%-   do engine.update({'name': engine.get('name', name) }) %}
+{%-   set index = engine.get('index', '1')~'-'~name %}
+{%-   do ext_engines.update({ index: engine }) %}
+{%- endfor %}
+{%- if ext_engines|length > 0 %}
 
 ext_pillar:
+  {%- for name, engine in ext_engines|dictsort %}
+  {%- if master.pillar.engine in ['composite', 'reclass'] and engine.name == 'reclass' %}
   - reclass: *reclass
+  {%- endif %}
+  {%- if master.pillar.engine in ['composite', 'saltclass'] and engine.name == 'saltclass' %}
+  - saltclass: *saltclass
+  {%- endif %}
+  {%- if engine.name == 'nacl' %}
+  - nacl: {}
+  {%- endif %}
+  {%- if engine.name == 'gpg' %}
+  - gpg: {}
+  {%- endif %}
+  {%- endfor %}
+{%- endif %}
+{%- endif %}
+
+{%- if master.pillar.engine in ['saltclass', 'reclass']
+    or (master.pillar.engine == 'composite' and
+        (master.pillar.saltclass is defined or
+         master.pillar.reclass is defined ))  %}
 
 master_tops:
+  {%- if master.pillar.engine in ['composite', 'reclass'] and master.pillar.get('reclass', False) %}
   reclass: *reclass
-
+  {%- endif %}
+  {%- if master.pillar.engine in ['composite', 'saltclass'] and master.pillar.get('saltclass', False) %}
+  saltclass: *saltclass
+  {%- endif %}
 {%- endif %}
 
 {%- for handler in pillar.salt.minion.get("handlers", []) %}
-
 {%- if handler.engine == "udp"%}
+
 logstash_udp_handler:
   host: {{ handler.host }}
   port: {{ handler.port }}
   version: 1
 {%- endif %}
-
 {%- if handler.engine == "zmq"%}
+
 logstash_zmq_handler:
   address: tcp://{{ handler.host }}:{{ handler.port }}
   version: 1
 {%- endif %}
-
 {%- endfor %}
 
 {%- if master.get('order_masters', False) %}
+
 order_masters: True
 {%- endif %}
 
+{%- if master.nacl is defined %}
+
+nacl.config:
+  box_type: {{ master.nacl.get('box_type', 'sealedbox') }}
+  {%- if master.nacl.sk is defined %}
+  sk: {{ master.nacl.sk }}
+  pk: {{ master.nacl.pk }}
+  {%- else %}
+  sk_file: {{ master.nacl.sk_file }}
+  pk_file: {{ master.nacl.pk_file }}
+  {%- endif %}
+{%- endif %}
+
 {#-
 vim: syntax=jinja
 -#}

salt/master/pillar.sls

-{%- from "salt/map.jinja" import master with context %}
+{%- from "salt/map.jinja" import master,storage with context %}
 {%- if master.enabled %}
 
 {%- if master.pillar.engine == 'salt' %}
 
 /srv/salt/reclass/classes/service:
   file.directory:
+  - makedirs: true
   - require:
     - file: reclass_data_dir
 
 
 /srv/salt/reclass/classes/service/{{ formula_name }}:
   file.symlink:
+  - makedirs: true
   - target: /srv/salt/env/{{ master.system.environment }}/{{ formula_name }}/metadata/service
   - require:
     - file: /srv/salt/reclass/classes/service
 
 {%- else %}
 
-{%- for environment_name, environment in master.environment.iteritems() %}
+{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
 
 {%- for formula_name, formula in environment.get('formula', {}).iteritems() %}
 
 
 /srv/salt/reclass/classes/service/{{ formula_name }}:
   file.symlink:
+  - makedirs: true
   {%- if formula.source == 'pkg' %}
   - target: /usr/share/salt-formulas/reclass/service/{{ formula_name }}
   {%- else %}

salt/master/service.sls

 salt_master_service:
   service.running:
   - name: {{ master.service }}
-  - enable: true
+  - enable: True
+  {%- if grains['saltversioninfo'][0] >= 2017 and grains['saltversioninfo'][1] >= 7 %}
+  - retry:
+    attempts: 2
+    interval: 5
+    splay: 5
+  {%- endif %}
 
 /srv/salt/env:
   file.directory:

tests/pillar/master_single_extpillars.sls

+git:
+  client:
+    enabled: true
+linux:
+  system:
+    enabled: true
+reclass:
+  storage:
+    enabled: true
+    data_source:
+      engine: git
+      branch: master
+      address: 'https://github.com/salt-formulas/openstack-salt.git'
+salt:
+  master:
+    enabled: true
+    command_timeout: 5
+    worker_threads: 2
+    base_environment: prd
+    #environment:
+    # prd:
+    #   formula:
+    #     python:
+    #       source: git
+    #       address: 'https://github.com/salt-formulas/salt-formula-python.git'
+    #       revision: master
+    pillar:
+      engine: composite
+      reclass:
+        # index: 1 is default value
+        index: 1
+        storage_type: yaml_fs
+        inventory_base_uri: /srv/salt/reclass_encrypted
+        class_mappings:
+          - target: '/^cfg\d+/'
+            class:  system.non-existing.class
+        ignore_class_notfound: True
+        ignore_class_regexp:
+          - 'service.*'
+          - '*.fluentd'
+        propagate_pillar_data_to_reclass: False
+      stack: # not yet implemented
+        # https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.stack.html
+        #option 1
+        #path:
+        #  - /path/to/stack.cfg
+        #option 2
+        pillar:environment:
+          dev: path/to/dev/stasck.cfg
+          prod: path/to/prod/stasck.cfg
+        grains:custom:grain:
+          value:
+            - /path/to/stack1.cfg
+            - /path/to/stack2.cfg
+      saltclass:
+        path: /srv/salt/saltclass
+      nacl:
+        # if order is provided 99 is used to compose "99-nacl" key name which is later used to order entries
+        index: 99
+      gpg: {}
+      vault-1: # not yet implemented
+        name: vault
+        path: secret/salt
+      vault-2: # not yet implemented
+        name: vault
+        path: secret/root
+    vault: # not yet implemented
+      # https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.vault.html
+      name: myvault
+      url: https://vault.service.domain:8200
+      auth:
+          method: token
+          token: 11111111-2222-3333-4444-555555555555
+      policies:
+          - saltstack/minions
+          - saltstack/minion/{minion}
+    nacl:
+      # https://docs.saltstack.com/en/develop/ref/modules/all/salt.modules.nacl.html
+      box_type: sealedbox
+      sk_file: /etc/salt/pki/master/nacl
+      pk_file: /etc/salt/pki/master/nacl.pub
+      #sk: None
+      #pk: None

tests/pillar/master_single_extreclass.sls

+git:
+  client:
+    enabled: true
+linux:
+  system:
+    enabled: true
+reclass:
+  storage:
+    enabled: true
+    data_source:
+      engine: git
+      branch: master
+      address: 'https://github.com/salt-formulas/openstack-salt.git'
+salt:
+  master:
+    enabled: true
+    command_timeout: 5
+    worker_threads: 2
+    base_environment: prd
+    #environment:
+    # prd:
+    #   formula:
+    #     python:
+    #       source: git
+    #       address: 'https://github.com/salt-formulas/salt-formula-python.git'
+    #       revision: master
+    pillar:
+      engine: reclass
+      reclass:
+        # index: 1 is default value
+        index: 1
+        storage_type: yaml_fs
+        inventory_base_uri: /srv/salt/reclass_encrypted
+        class_mappings:
+          - target: '/^cfg\d+/'
+            class:  system.non-existing.class
+        ignore_class_notfound: True
+        ignore_class_regexp:
+          - 'service.*'
+          - '*.fluentd'
+        propagate_pillar_data_to_reclass: False

tests/pillar/master_single_reclass.sls

             name: salt-formula-service02
     pillar:
       engine: reclass
-      data_dir: /srv/salt/reclass
+      reclass:
+        storage_type: yaml_fs
+        inventory_base_uri: /srv/salt/reclass
+        propagate_pillar_data_to_reclass: False
+        reclass_source_path: /tmp/reclass