Merge branch 'github' into itg

Conflicts:
	salt/formulas.jinja

salt/api.sls

@@ -9,8 +9,11 @@ salt-api:
     - name: {{ salt_settings.salt_api }}
 {% endif %}
   service.running:
+    - enable: True
     - name: {{ salt_settings.api_service }}
     - require:
       - service: {{ salt_settings.master_service }}
+{% if salt_settings.install_packages %}
     - watch:
-      - pkg: salt-master
+      - pkg: salt-api
+{% endif %}

salt/defaults.yaml

@@ -28,6 +28,17 @@ salt:
       install_from_source: True
     pygit2:
       install_from_source: True
+      version: 0.23.0
+      git:
+        # if not false, should be state name
+        require_state: False
+        install_from_package: git
+      libgit2:
+        version: 0.23.0
+        install_from_source: True
+        build_parent_dir: /usr/src/
+        # hash necessary until github issue #9272 is addressed
+        download_hash: 683d1164e361e2a0a8d52652840e2340
     gitpython:
       install_from_source: False
 

salt/files/master.d/f_defaults.conf

@@ -109,6 +109,9 @@
 # Return minions that timeout when running commands like test.ping
 {{ get_config('show_timeout', 'True') }}
 
+# Display the jid when a job is published
+{{ get_config('show_jid', 'False') }}
+
 # By default, output is colored. To disable colored output, set the color value
 # to False.
 {{ get_config('color', 'True') }}
@@ -135,7 +138,7 @@
 {{ get_config('minion_data_cache', 'True') }}
 
 # Store all returns in the given returner.
-# Setting this option requires that any returner-specific configuration also 
+# Setting this option requires that any returner-specific configuration also
 # be set. See various returners in salt/returners for details on required
 # configuration values. (See also, event_return_queue below.)
 #
@@ -199,12 +202,12 @@ event_return_blacklist:
 # the key rotation event as minions reconnect. Consider this carefully if this
 # salt master is managing a large number of minions.
 #
-# If disabled, it is recommended to handle this event by listening for the 
+# If disabled, it is recommended to handle this event by listening for the
 # 'aes_key_rotate' event with the 'key' tag and acting appropriately.
 {{ get_config('ping_on_rotate', 'False') }}
 
 # By default, the master deletes its cache of minion data when the key for that
-# minion is removed. To preserve the cache after key deletion, set 
+# minion is removed. To preserve the cache after key deletion, set
 # 'preserve_minion_cache' to True.
 #
 # WARNING: This may have security implications if compromised minions auth with
@@ -336,7 +339,7 @@ client_acl_blacklist:
 {% endif %}
 
 # Enforce client_acl & client_acl_blacklist when users have sudo
-# access to the salt command. 
+# access to the salt command.
 #
 {{ get_config('sudo_acl', 'False') }}
 
@@ -405,7 +408,19 @@ client_acl_blacklist:
 # master_tops:
 #   ext_nodes: <Shell command which returns yaml>
 #
-{{ get_config('master_tops', '{}') }}
+{% if 'master_tops' in cfg_master %}
+master_tops:
+{%- for master in cfg_master['master_tops'] -%}
+  {%- if cfg_master['master_tops'][master] is string %}
+  {{ master }}: {{ cfg_master['master_tops'][master] }}
+  {%- else %}
+  {{ master}}:
+    {%- for parameter in cfg_master['master_tops'][master] %}
+    {{ parameter }}: {{ cfg_master['master_tops'][master][parameter] }}
+    {%- endfor -%}
+  {%- endif -%}
+{%- endfor %}
+{% endif %}
 
 # The external_nodes option allows Salt to gather data that would normally be
 # placed in a top file. The external_nodes option is the executable that will
@@ -492,7 +507,7 @@ client_acl_blacklist:
 # the master server. The default is md5, but sha1, sha224, sha256, sha384
 # and sha512 are also supported.
 #
-# Prior to changing this value, the master should be stopped and all Salt 
+# Prior to changing this value, the master should be stopped and all Salt
 # caches should be cleared.
 {{ get_config('hash_type', 'md5') }}
 
@@ -652,12 +667,12 @@ pillar_roots:
 {%- for dir in roots %}
     - {{ dir }}
 {%- endfor -%}
-{% endfor %}
-{%- elif 'pillar_roots' in cfg_salt -%}
+{%- endfor -%}
+{% elif 'pillar_roots' in cfg_salt -%}
 pillar_roots:
-{%- for name, roots in cfg_salt['pillar_roots']|dictsort -%}
+{%- for name, roots in cfg_salt['pillar_roots']|dictsort %}
   {{ name }}:
-{%- for dir in roots -%}
+{%- for dir in roots %}
     - {{ dir }}
 {%- endfor -%}
 {%- endfor -%}
@@ -669,9 +684,25 @@ pillar_roots:
 #
 {% if 'ext_pillar' in cfg_master %}
 ext_pillar:
-{% for pillar in cfg_master['ext_pillar'] %}
-  - {{ pillar.items()[0][0] }}: {{ pillar.items()[0][1] }}
-{% endfor %}
+{%- for pillar in cfg_master['ext_pillar'] -%}
+  {%- for key in pillar -%}
+    {%- if pillar[key] is string %}
+  - {{ key }}: {{ pillar[key] }}
+    {%- elif pillar[key] is iterable and pillar[key] is not mapping %}
+  - {{ key }}:
+      {%- for parameter in pillar[key] %}
+    - {{ parameter }}
+      {%- endfor -%}
+    {%- elif pillar[key] is mapping and pillar[key] is not string %}
+  - {{ key }}: 
+      {%- for parameter in pillar[key] %}
+      {{ parameter }}: {{pillar[key][parameter]}}
+      {%- endfor %}
+    {%- else %}
+# Error in rendering {{ key }}, please read https://docs.saltstack.com/en/latest/topics/development/external_pillars.html#configuration
+    {% endif %}
+  {%- endfor -%}
+{%- endfor %}
 {% elif 'ext_pillar' in cfg_salt %}
 ext_pillar:
 {% for pillar in cfg_salt['ext_pillar'] %}

salt/files/minion.d/f_defaults.conf

@@ -20,8 +20,7 @@
 {{ get_config('default_include', 'minion.d/*.conf') }}
 
 # master configs
-{%- if 'master' in cfg_minion -%}
-{%- if cfg_minion['master'] is not string %}
+{%- if 'master' in cfg_minion and cfg_minion['master'] is not string %}
 master:
   {% for name in cfg_minion['master'] -%}
   - {{ name }}
@@ -29,16 +28,6 @@ master:
 {%- else %}
 {{ get_config('master', 'salt') }}
 {%- endif %}
-{% elif 'master' in cfg_salt -%}
-{%- if cfg_salt['master'] is not string %}
-master:
-  {% for name in cfg_salt['master'] -%}
-  - {{ name }}
-  {% endfor -%}
-{%- else %}
-{{ get_config('master', 'salt') }}
-{%- endif -%}
-{%- endif %}
 
 # choose a random master
 {{ get_config('random_master', 'False') }}
@@ -157,6 +146,7 @@ mine_functions:
 
 # verify_master_pubkey_sign
 {{ get_config('verify_master_pubkey_sign', 'False') }}
+{{ get_config('master_type', 'str') }}
 
 # include extra config
 {% if 'include' in cfg_minion -%}

salt/formulas.jinja

@@ -11,17 +11,14 @@
 {%- set value = salt['pillar.get']('salt_formulas:git_opts:{0}:{1}'.format(env, opt),
      salt['pillar.get']('salt_formulas:git_opts:default:{0}'.format(opt),
        defaults[opt])) -%}
-{%- if value is iterable -%}
 {{ value|yaml }}
-{%- else -%}
-{{ value }}
-{%- endif -%}
 {%- endmacro -%}
 
 {%- macro formulas_roots(env) -%}
 {%- set value = [] -%}
 {%- for dir in formulas.get(env, []) -%}
-{%- do value.append('{0}/{1}'.format(formulas_git_opt(env, 'basedir'), dir)) -%}
+{%- set basedir = formulas_git_opt(env, 'basedir')|load_yaml -%}
+{%- do value.append('{0}/{1}'.format(basedir, dir)) -%}
 {%- endfor -%}
 {{ value|yaml }}
 {%- endmacro -%}

salt/formulas.sls

@@ -4,10 +4,10 @@
 {% from "salt/formulas.jinja" import formulas_git_opt with context %}
 
 # Loop over all formulas listed in pillar data
-{% for env, entries in salt['pillar.get']('salt_formulas:list', {}).iteritems() %}
+{% for env, entries in salt['pillar.get']('salt_formulas:list', {}).items() %}
 {% for entry in entries %}
 
-{% set basedir = formulas_git_opt(env, 'basedir') %}
+{% set basedir = formulas_git_opt(env, 'basedir')|load_yaml %}
 {% set gitdir = '{0}/{1}'.format(basedir, entry) %}
 {% set update = formulas_git_opt(env, 'update')|load_yaml %}
 
@@ -17,7 +17,7 @@
 {{ basedir }}:
   file.directory:
     {%- for key, value in salt['pillar.get']('salt_formulas:basedir_opts',
-                                             {'makedirs': True}).iteritems() %}
+                                             {'makedirs': True}).items() %}
     - {{ key }}: {{ value }}
     {%- endfor %}
 {% endif %}
@@ -26,11 +26,12 @@
 {% if gitdir not in processed_gitdirs %}
 {% do processed_gitdirs.append(gitdir) %}
 {% set options = formulas_git_opt(env, 'options')|load_yaml %}
+{% set baseurl = formulas_git_opt(env, 'baseurl')|load_yaml %}
 {{ gitdir }}:
   git.latest:
-    - name: {{ formulas_git_opt(env, 'baseurl') }}/{{ entry }}.git
+    - name: {{ baseurl }}/{{ entry }}.git
     - target: {{ gitdir }}
-    {%- for key, value in options.iteritems() %}
+    {%- for key, value in options.items() %}
     - {{ key }}: {{ value }}
     {%- endfor %}
     - require:

salt/gitfs/pygit2.sls

@@ -1,9 +1,22 @@
 {% from "salt/map.jinja" import salt_settings with context %}
+{% set pygit2_settings = salt_settings.gitfs.pygit2 %}
 
-git:
-  pkg.installed
+{% if pygit2_settings.git.get('require_state', False) %}
+include:
+  - {{ pygit2_settings.git.require_state }}
+{% elif pygit2_settings.git.get('install_from_package', 'git') %}
+pygit2-git:
+  pkg.installed:
+    - name: {{ pygit2_settings.git.install_from_package }}
+{% endif %}
+
+{% if pygit2_settings.install_from_source %}
+{% set libgit2_settings = pygit2_settings.libgit2 %}
+
+{% if libgit2_settings.install_from_source %}
+{% set libgit2_src_dir = libgit2_settings.build_parent_dir + 'libgit2-' + libgit2_settings.version %}
+{% set libgit2_build_dir = libgit2_src_dir + '/_build' %}
 
-{% if salt_settings.gitfs.pygit2.install_from_source %}
 # we probably don't have a package or it's not a high enough version
 # install latest from source/pip
 pygit-deps:
@@ -17,32 +30,32 @@ pygit-deps:
 
 dl-libgit2-src:
   archive.extracted:
-    - name: /usr/src
-    - source: https://github.com/libgit2/libgit2/archive/v0.22.1.tar.gz
-    - source_hash: md5=dbf516d18e176bbb131de3efccfee533
+    - name: {{ libgit2_settings.build_parent_dir }}
+    - source: https://github.com/libgit2/libgit2/archive/v{{ libgit2_settings.version }}.tar.gz
+    - source_hash: md5={{ libgit2_settings.download_hash }}
     - archive_format: tar
     - keep: True
-    - if_missing: /usr/src/libgit2-0.22.1
+    - if_missing: /usr/src/libgit2-{{ libgit2_settings.version }}
 
-/usr/src/libgit2-0.22.1/_build:
+{{ libgit2_build_dir }}:
   file.directory
 
 configure-libgit2:
   cmd.run:
     - name: cmake ..
-    - cwd: /usr/src/libgit2-0.22.1/_build
-    - creates: /usr/src/libgit2-0.22.1/_build/Makefile
+    - cwd: {{ libgit2_build_dir }}
+    - creates: {{ libgit2_build_dir }}/Makefile
 
 build-libgit2:
   cmd.run:
     - name: make -j4
-    - cwd: /usr/src/libgit2-0.22.1/_build
-    - creates: /usr/src/libgit2-0.22.1/_build/libgit2.so
+    - cwd: {{ libgit2_build_dir }}
+    - creates: {{ libgit2_build_dir }}/libgit2.so
 
 install-libgit2:
   cmd.run:
     - name: make install
-    - cwd: /usr/src/libgit2-0.22.1/_build
+    - cwd: {{ libgit2_build_dir }}
     - creates: /usr/local/lib/libgit2.so
 
 run-ldconfig-after-lib-install:
@@ -51,9 +64,15 @@ run-ldconfig-after-lib-install:
     - onchanges:
       - cmd: install-libgit2
 
+{% else %}
+{{ salt_settings.libgit2 }}:
+  pkg.installed
+
+{% endif %}
+
 install-pygit2:
   pip.installed:
-    - name: pygit2
+    - name: pygit2 == {{ pygit2_settings.version }}
 
 {% else %}
 {{ salt_settings.pygit2 }}:

salt/map.jinja

@@ -4,27 +4,44 @@
 {## Start with  defaults from defaults.yaml ##}
 {% import_yaml "salt/defaults.yaml" as default_settings %}
 
-{## 
+{##
 Setup variable using grains['os_family'] based logic, only add key:values here
 that differ from whats in defaults.yaml
 ##}
 {% set os_family_map = salt['grains.filter_by']({
-    'Debian':  {},
-    'Ubuntu':  {},
-    'CentOS':  {},
-    'Amazon':  {},
-    'Fedora':  {},
+    'Debian':  {
+      'pkgrepo': 'deb http://debian.saltstack.com/debian ' + salt['grains.get']('oscodename') + '-saltstack main',
+      'key_url': 'salt://' + slspath + '/saltstack.gpg',
+      'libgit2': 'libgit2-22',
+      'gitfs': {
+        'pygit2': {
+          'install_from_source': True,
+          'version': '0.22.1',
+          'git': {
+            'require_state': False,
+            'install_from_package': 'git',
+          },
+          'libgit2': {
+            'install_from_source': False,
+          },
+        },
+      },
+    },
     'RedHat':  {
       'pygit2': 'python-pygit2',
       'gitfs': {
         'pygit2': {
-           'install_from_source': False
+          'install_from_source': False,
+          'git': {
+            'require_state': False,
+            'install_from_package': 'git',
+          },
         },
       },
       'master': {
         'gitfs_provider': 'pygit2'
       },
-      'repotype': 'epel'
+      'repotype': 'epel',
     },
     'Suse':    {},
     'Gentoo':  {
@@ -32,7 +49,7 @@ that differ from whats in defaults.yaml
       'salt_minion': 'app-admin/salt',
       'salt_syndic': 'app-admin/salt',
       'salt_api': 'app-admin/salt',
-      'salt_cloud': 'app-admin/salt'
+      'salt_cloud': 'app-admin/salt',
     },
     'Arch':    {
       'salt_master': 'salt-zmq',
@@ -40,7 +57,7 @@ that differ from whats in defaults.yaml
       'salt_syndic':  'salt-zmq',
       'salt_cloud':  'salt-zmq',
       'salt_api': 'salt-zmq',
-      'salt_ssh':  'salt-zmq'
+      'salt_ssh':  'salt-zmq',
     },
     'FreeBSD': {
       'salt_master': 'py27-salt',
@@ -55,20 +72,17 @@ that differ from whats in defaults.yaml
       'minion_service': 'salt_minion',
       'master_service': 'salt_master',
       'api_service': 'salt_api',
-      'syndic_service': 'salt_syndic'
+      'syndic_service': 'salt_syndic',
     },
-  }
-  , grain="os_family"
-  , merge=salt['pillar.get']('salt:lookup'))
+  }, grain="os_family", merge=salt['pillar.get']('salt:lookup'))
 %}
 
 {## Merge the flavor_map to the default settings ##}
 {% do default_settings.salt.update(os_family_map) %}
-    
+
 {## Merge in salt:lookup pillar ##}
 {% set salt_settings = salt['pillar.get'](
     'salt',
     default=default_settings.salt,
-    merge=True
-  )
+    merge=True)
 %}

salt/pkgrepo/debian/absent.sls

@@ -1,4 +1,8 @@
+{% from "salt/map.jinja" import salt_settings with context %}
+
 drop-saltstack-pkgrepo:
+  pkgrepo.absent:
+    - name: {{ salt_settings.pkgrepo }}
   file.absent:
     - name: /etc/apt/sources.list.d/saltstack.list
 

salt/pkgrepo/debian/init.sls

@@ -1,21 +1,12 @@
-saltstack-apt-key:
-  file.managed:
-    - name: /etc/apt/trusted.gpg.d/saltstack.gpg
-    - source: salt://{{ slspath }}/saltstack.gpg
-    - user: root
-    - group: root
-    - mode: 644
+{% from "salt/map.jinja" import salt_settings with context %}
 
 saltstack-pkgrepo:
-  file.managed:
-    - name: /etc/apt/sources.list.d/saltstack.list
-    - source: salt://{{ slspath }}/sources.list
-    - user: root
-    - group: root
-    - mode: 644
-    - template: jinja
-    - require:
-      - file: saltstack-apt-key
+  pkgrepo.managed:
+    - humanname: SaltStack Debian Repo
+    - name: {{ salt_settings.pkgrepo }}
+    - file: /etc/apt/sources.list.d/saltstack.list
+    - key_url: {{ salt_settings.key_url }}
+    - clean_file: True
     # Order: 1 because we can't put a require_in on "pkg: salt-{master,minion}"
     # because we don't know if they are used.
     - order: 1

salt/pkgrepo/debian/sources.list

@@ -1,2 +0,0 @@
-# saltstack
-deb http://debian.saltstack.com/debian {{ grains['oscodename'] }}-saltstack main

salt/syndic.sls

@@ -9,6 +9,7 @@ salt-syndic:
     - name: {{ salt_settings.salt_syndic }}
 {% endif %}
   service.running:
+    - name: {{ salt_settings.syndic_service }}
     - require:
       - service: salt-master
     - watch: